From 2ebdcbc3b0c1da438889c9cce5f38bf92bc031a0 Mon Sep 17 00:00:00 2001 From: stack Date: Tue, 1 Oct 2019 09:24:10 -0700 Subject: [PATCH] Revert " HBASE-23055 Alter hbase:meta (#667)" Revert because new feedback and requested survey of master usage figuring table state. This reverts commit 52176184df595cf591e7c13fe18d959b2ba039a8. --- .../hadoop/hbase/AsyncMetaTableAccessor.java | 3 +- .../hadoop/hbase/MetaTableAccessor.java | 20 +-- .../hbase/client/RawAsyncHBaseAdmin.java | 50 +++--- .../hadoop/hbase/client/ZKAsyncRegistry.java | 12 +- .../hadoop/hbase/zookeeper/ZNodePaths.java | 59 +++---- .../org/apache/hadoop/hbase/HConstants.java | 3 +- .../apache/hadoop/hbase/TableDescriptors.java | 20 ++- .../apache/hadoop/hbase/master/HMaster.java | 7 +- .../hbase/master/TableStateManager.java | 115 +++++++++---- .../master/assignment/RegionStateStore.java | 3 +- .../procedure/CreateTableProcedure.java | 4 +- .../procedure/DisableTableProcedure.java | 10 +- .../procedure/EnableTableProcedure.java | 152 +++++++++--------- .../replication/AbstractPeerProcedure.java | 6 +- .../replication/ModifyPeerProcedure.java | 6 +- .../master/zksyncer/MetaLocationSyncer.java | 6 +- .../hbase/regionserver/HRegionServer.java | 7 +- .../RejectReplicationRequestStateChecker.java | 5 +- .../hadoop/hbase/util/FSTableDescriptors.java | 148 ++++++++++------- .../hadoop/hbase/HBaseTestingUtility.java | 13 +- .../hadoop/hbase/TestHBaseMetaEdit.java | 109 ------------- .../hadoop/hbase/client/TestAdmin2.java | 17 ++ .../TestAsyncAccessControlAdminApi.java | 2 +- .../hbase/client/TestAsyncTableAdminApi2.java | 13 ++ .../hbase/client/TestAsyncTableAdminApi3.java | 12 ++ .../hbase/client/TestMetaWithReplicas.java | 7 - .../hbase/regionserver/TestClusterId.java | 2 +- .../TestGetClosestAtOrBefore.java | 5 +- .../wal/TestLogRollingNoCluster.java | 8 +- .../hbase/util/TestFSTableDescriptors.java | 7 +- .../hbase/zookeeper/MetaTableLocator.java | 18 ++- .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 2 +- 32 files changed, 444 insertions(+), 407 deletions(-) delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseMetaEdit.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/AsyncMetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/AsyncMetaTableAccessor.java index bc769ba0260e..d04ea527f747 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/AsyncMetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/AsyncMetaTableAccessor.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -616,5 +616,4 @@ private static int parseReplicaIdFromServerColumn(byte[] serverColumn) { } return -1; } - } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 3ad2a068deee..ad54324c7b3d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -303,18 +303,11 @@ public static HRegionLocation getRegionLocation(Connection connection, byte[] re */ public static HRegionLocation getRegionLocation(Connection connection, RegionInfo regionInfo) throws IOException { - return getRegionLocation(getCatalogFamilyRow(connection, regionInfo), - regionInfo, regionInfo.getReplicaId()); - } - - /** - * @return Return the {@link HConstants#CATALOG_FAMILY} row from hbase:meta. - */ - public static Result getCatalogFamilyRow(Connection connection, RegionInfo ri) - throws IOException { - Get get = new Get(getMetaKeyForRegion(ri)); + byte[] row = getMetaKeyForRegion(regionInfo); + Get get = new Get(row); get.addFamily(HConstants.CATALOG_FAMILY); - return get(getMetaHTable(connection), get); + Result r = get(getMetaHTable(connection), get); + return getRegionLocation(r, regionInfo, regionInfo.getReplicaId()); } /** Returns the row key to use for this regionInfo */ @@ -1084,7 +1077,7 @@ public static RegionInfo getRegionInfo(final Result r, byte [] qualifier) { public static TableState getTableState(Connection conn, TableName tableName) throws IOException { if (tableName.equals(TableName.META_TABLE_NAME)) { - throw new IllegalAccessError("Go to the Master to find hbase:meta table state, not here"); + return new TableState(tableName, TableState.State.ENABLED); } Table metaHTable = getMetaHTable(conn); Get get = new Get(tableName.getName()).addColumn(getTableFamily(), getTableStateColumn()); @@ -1112,8 +1105,7 @@ public static Map getTableStates(Connection conn) } /** - * Updates state in META. - * Do not use. For internal use only. + * Updates state in META * @param conn connection to use * @param tableName table to look for */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 5f693027aa97..92cc0dcc929b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -90,7 +90,6 @@ import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil; import org.apache.hadoop.hbase.security.access.UserPermission; - import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; @@ -193,8 +192,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableDescriptorsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableNamesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableNamesResponse; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableStateRequest; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableStateResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsBalancerEnabledRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsBalancerEnabledResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledRequest; @@ -669,25 +666,42 @@ public CompletableFuture disableTable(TableName tableName) { @Override public CompletableFuture isTableEnabled(TableName tableName) { - return isTableState(tableName, TableState.State.ENABLED); + if (TableName.isMetaTableName(tableName)) { + return CompletableFuture.completedFuture(true); + } + CompletableFuture future = new CompletableFuture<>(); + addListener(AsyncMetaTableAccessor.getTableState(metaTable, tableName), (state, error) -> { + if (error != null) { + future.completeExceptionally(error); + return; + } + if (state.isPresent()) { + future.complete(state.get().inStates(TableState.State.ENABLED)); + } else { + future.completeExceptionally(new TableNotFoundException(tableName)); + } + }); + return future; } @Override public CompletableFuture isTableDisabled(TableName tableName) { - return isTableState(tableName, TableState.State.DISABLED); - } - - /** - * @return Future that calls Master getTableState and compares to state - */ - private CompletableFuture isTableState(TableName tableName, TableState.State state) { - return this. newMasterCaller(). - action((controller, stub) -> - this. call(controller, stub, - GetTableStateRequest.newBuilder(). - setTableName(ProtobufUtil.toProtoTableName(tableName)).build(), - (s, c, req, done) -> s.getTableState(c, req, done), - resp -> resp.getTableState().getState().toString().equals(state.toString()))).call(); + if (TableName.isMetaTableName(tableName)) { + return CompletableFuture.completedFuture(false); + } + CompletableFuture future = new CompletableFuture<>(); + addListener(AsyncMetaTableAccessor.getTableState(metaTable, tableName), (state, error) -> { + if (error != null) { + future.completeExceptionally(error); + return; + } + if (state.isPresent()) { + future.complete(state.get().inStates(TableState.State.DISABLED)); + } else { + future.completeExceptionally(new TableNotFoundException(tableName)); + } + }); + return future; } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKAsyncRegistry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKAsyncRegistry.java index 0a021aa0f720..36fa6bba7544 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKAsyncRegistry.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZKAsyncRegistry.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -158,8 +158,7 @@ private void getMetaRegionLocation(CompletableFuture future, } Pair stateAndServerName = getStateAndServerName(proto); if (stateAndServerName.getFirst() != RegionState.State.OPEN) { - LOG.warn("hbase:meta region (replicaId={}) is in state {}", replicaId, - stateAndServerName.getFirst()); + LOG.warn("Meta region is in state " + stateAndServerName.getFirst()); } locs[DEFAULT_REPLICA_ID] = new HRegionLocation( getRegionInfoForDefaultReplica(FIRST_META_REGIONINFO), stateAndServerName.getSecond()); @@ -174,7 +173,7 @@ private void getMetaRegionLocation(CompletableFuture future, LOG.warn("Failed to fetch " + path, error); locs[replicaId] = null; } else if (proto == null) { - LOG.warn("hbase:meta znode for replica " + replicaId + " is null"); + LOG.warn("Meta znode for replica " + replicaId + " is null"); locs[replicaId] = null; } else { Pair stateAndServerName = getStateAndServerName(proto); @@ -198,8 +197,9 @@ private void getMetaRegionLocation(CompletableFuture future, public CompletableFuture getMetaRegionLocation() { CompletableFuture future = new CompletableFuture<>(); addListener( - zk.list(znodePaths.baseZNode).thenApply(children -> children.stream(). - filter(c -> znodePaths.isMetaZNodePrefix(c)).collect(Collectors.toList())), + zk.list(znodePaths.baseZNode) + .thenApply(children -> children.stream() + .filter(c -> c.startsWith(znodePaths.metaZNodePrefix)).collect(Collectors.toList())), (metaReplicaZNodes, error) -> { if (error != null) { future.completeExceptionally(error); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java index cfa0e82ea0e4..c5e510fe4b9c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,7 +24,6 @@ import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT; import static org.apache.hadoop.hbase.client.RegionInfo.DEFAULT_REPLICA_ID; -import java.util.Collection; import java.util.Optional; import java.util.stream.IntStream; import org.apache.hadoop.conf.Configuration; @@ -41,24 +40,15 @@ public class ZNodePaths { // TODO: Replace this with ZooKeeper constant when ZOOKEEPER-277 is resolved. public static final char ZNODE_PATH_SEPARATOR = '/'; - private static final String META_ZNODE_PREFIX = "meta-region-server"; + public final static String META_ZNODE_PREFIX = "meta-region-server"; private static final String DEFAULT_SNAPSHOT_CLEANUP_ZNODE = "snapshot-cleanup"; // base znode for this cluster public final String baseZNode; - - /** - * The prefix of meta znode. Does not include baseZNode. - * Its a 'prefix' because meta replica id integer can be tagged on the end (if - * no number present, it is 'default' replica). - */ - private final String metaZNodePrefix; - - /** - * znodes containing the locations of the servers hosting the meta replicas - */ - private final ImmutableMap metaReplicaZNodes; - + // the prefix of meta znode, does not include baseZNode. + public final String metaZNodePrefix; + // znodes containing the locations of the servers hosting the meta replicas + public final ImmutableMap metaReplicaZNodes; // znode containing ephemeral nodes of the regionservers public final String rsZNode; // znode containing ephemeral nodes of the draining regionservers @@ -168,21 +158,21 @@ public String toString() { } /** - * @return true if the znode is a meta region replica + * Is the znode of any meta replica + * @param node + * @return true or false */ public boolean isAnyMetaReplicaZNode(String node) { - return this.metaReplicaZNodes.containsValue(node); - } - - /** - * @return Meta Replica ZNodes - */ - public Collection getMetaReplicaZNodes() { - return this.metaReplicaZNodes.values(); + if (metaReplicaZNodes.containsValue(node)) { + return true; + } + return false; } /** - * @return the znode string corresponding to a replicaId + * Get the znode string corresponding to a replicaId + * @param replicaId + * @return znode */ public String getZNodeForReplica(int replicaId) { // return a newly created path but don't update the cache of paths @@ -193,21 +183,24 @@ public String getZNodeForReplica(int replicaId) { } /** - * Parse the meta replicaId from the passed znode name. + * Parse the meta replicaId from the passed znode * @param znode the name of the znode, does not include baseZNode * @return replicaId */ public int getMetaReplicaIdFromZnode(String znode) { - return znode.equals(metaZNodePrefix)? - RegionInfo.DEFAULT_REPLICA_ID: - Integer.parseInt(znode.substring(metaZNodePrefix.length() + 1)); + if (znode.equals(metaZNodePrefix)) { + return RegionInfo.DEFAULT_REPLICA_ID; + } + return Integer.parseInt(znode.substring(metaZNodePrefix.length() + 1)); } /** - * @return True if meta znode. + * Is it the default meta replica's znode + * @param znode the name of the znode, does not include baseZNode + * @return true or false */ - public boolean isMetaZNodePrefix(String znode) { - return znode != null && znode.startsWith(this.metaZNodePrefix); + public boolean isDefaultMetaReplicaZnode(String znode) { + return metaReplicaZNodes.get(DEFAULT_REPLICA_ID).equals(znode); } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index d810ba406e7d..a999e73b02eb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -1175,7 +1175,8 @@ public enum OperationStatusCode { /** Directories that are not HBase user table directories */ public static final List HBASE_NON_USER_TABLE_DIRS = Collections.unmodifiableList(Arrays.asList((String[])ArrayUtils.addAll( - HBASE_NON_TABLE_DIRS.toArray()))); + new String[] { TableName.META_TABLE_NAME.getNameAsString() }, + HBASE_NON_TABLE_DIRS.toArray()))); /** Health script related settings. */ public static final String HEALTH_SCRIPT_LOC = "hbase.node.health.script.location"; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java index e0a9eabfa629..2537e7f83e79 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java @@ -25,19 +25,25 @@ /** * Get, remove and modify table descriptors. + * Used by servers to host descriptors. */ @InterfaceAudience.Private public interface TableDescriptors { /** + * @param tableName * @return TableDescriptor for tablename + * @throws IOException */ - TableDescriptor get(final TableName tableName) throws IOException; + TableDescriptor get(final TableName tableName) + throws IOException; /** * Get Map of all NamespaceDescriptors for a given namespace. * @return Map of all descriptors. + * @throws IOException */ - Map getByNamespace(String name) throws IOException; + Map getByNamespace(String name) + throws IOException; /** * Get Map of all TableDescriptors. Populates the descriptor cache as a @@ -45,19 +51,25 @@ public interface TableDescriptors { * Notice: the key of map is the table name which contains namespace. It was generated by * {@link TableName#getNameWithNamespaceInclAsString()}. * @return Map of all descriptors. + * @throws IOException */ Map getAll() throws IOException; /** * Add or update descriptor * @param htd Descriptor to set into TableDescriptors + * @throws IOException */ - void add(final TableDescriptor htd) throws IOException; + void add(final TableDescriptor htd) + throws IOException; /** + * @param tablename * @return Instance of table descriptor or null if none found. + * @throws IOException */ - TableDescriptor remove(final TableName tablename) throws IOException; + TableDescriptor remove(final TableName tablename) + throws IOException; /** * Enables the tabledescriptor cache diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index f5a9ae335615..98841f95c838 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -49,6 +49,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.servlet.ServletException; @@ -669,6 +670,10 @@ private int putUpJettyServer() throws IOException { return connector.getLocalPort(); } + @Override + protected Function getMetaTableObserver() { + return builder -> builder.setRegionReplication(conf.getInt(HConstants.META_REPLICAS_NUM, HConstants.DEFAULT_META_REPLICA_NUM)); + } /** * For compatibility, if failed with regionserver credentials, try the master one */ @@ -1030,7 +1035,7 @@ private void finishActiveMasterInitialization(MonitoredTask status) throws IOExc RegionState rs = this.assignmentManager.getRegionStates(). getRegionState(RegionInfoBuilder.FIRST_META_REGIONINFO); LOG.info("hbase:meta {}", rs); - if (rs != null && rs.isOffline()) { + if (rs.isOffline()) { Optional optProc = procedureExecutor.getProcedures().stream() .filter(p -> p instanceof InitMetaProcedure).map(o -> (InitMetaProcedure) o).findAny(); initMetaProc = optProc.orElseGet(() -> { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java index 5901ae2f6606..1eb041692082 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException; import org.apache.hadoop.hbase.util.IdReadWriteLock; import org.apache.hadoop.hbase.util.ZKDataMigrator; import org.apache.hadoop.hbase.zookeeper.ZKUtil; @@ -52,20 +53,8 @@ // TODO: Make this a guava Service @InterfaceAudience.Private public class TableStateManager { - private static final Logger LOG = LoggerFactory.getLogger(TableStateManager.class); - - /** - * All table state is kept in hbase:meta except that of hbase:meta itself. - * hbase:meta state is kept here locally in this in-memory variable. State - * for hbase:meta is not persistent. If this process dies, the hbase:meta - * state reverts to enabled. State is used so we can edit hbase:meta as we - * would any other table by disabling, altering, and then re-enabling. If this - * process dies in the midst of an edit, the table reverts to enabled. Schema - * is read from the filesystem. It is changed atomically so if we die midway - * through an edit we should be good. - */ - private TableState.State metaTableState = TableState.State.ENABLED; + private static final Logger LOG = LoggerFactory.getLogger(TableStateManager.class); /** * Set this key to false in Configuration to disable migrating table state from zookeeper so * hbase:meta table. @@ -79,7 +68,7 @@ public class TableStateManager { private final ConcurrentMap tableName2State = new ConcurrentHashMap<>(); - TableStateManager(MasterServices master) { + public TableStateManager(MasterServices master) { this.master = master; } @@ -98,6 +87,61 @@ public void setTableState(TableName tableName, TableState.State newState) throws } } + /** + * Set table state to provided but only if table in specified states Caller should lock table on + * write. + * @param tableName table to change state for + * @param newState new state + * @param states states to check against + * @return null if succeed or table state if failed + */ + public TableState setTableStateIfInStates(TableName tableName, TableState.State newState, + TableState.State... states) throws IOException { + ReadWriteLock lock = tnLock.getLock(tableName); + lock.writeLock().lock(); + try { + TableState currentState = readMetaState(tableName); + if (currentState == null) { + throw new TableNotFoundException(tableName); + } + if (currentState.inStates(states)) { + updateMetaState(tableName, newState); + return null; + } else { + return currentState; + } + } finally { + lock.writeLock().unlock(); + } + } + + /** + * Set table state to provided but only if table not in specified states Caller should lock table + * on write. + * @param tableName table to change state for + * @param newState new state + * @param states states to check against + */ + public boolean setTableStateIfNotInStates(TableName tableName, TableState.State newState, + TableState.State... states) throws IOException { + ReadWriteLock lock = tnLock.getLock(tableName); + lock.writeLock().lock(); + try { + TableState currentState = readMetaState(tableName); + if (currentState == null) { + throw new TableNotFoundException(tableName); + } + if (!currentState.inStates(states)) { + updateMetaState(tableName, newState); + return true; + } else { + return false; + } + } finally { + lock.writeLock().unlock(); + } + } + public boolean isTableState(TableName tableName, TableState.State... states) { try { TableState tableState = getTableState(tableName); @@ -111,7 +155,6 @@ public boolean isTableState(TableName tableName, TableState.State... states) { public void setDeletedTable(TableName tableName) throws IOException { if (tableName.equals(TableName.META_TABLE_NAME)) { - // Can't delete the hbase:meta table. return; } ReadWriteLock lock = tnLock.getLock(tableName); @@ -140,7 +183,7 @@ public boolean isTablePresent(TableName tableName) throws IOException { * @param states filter by states * @return tables in given states */ - Set getTablesInStates(TableState.State... states) throws IOException { + public Set getTablesInStates(TableState.State... states) throws IOException { // Only be called in region normalizer, will not use cache. final Set rv = Sets.newHashSet(); MetaTableAccessor.fullScanTables(master.getConnection(), new MetaTableAccessor.Visitor() { @@ -156,6 +199,12 @@ public boolean visit(Result r) throws IOException { return rv; } + public static class TableStateNotFoundException extends TableNotFoundException { + TableStateNotFoundException(TableName tableName) { + super(tableName.getNameAsString()); + } + } + @NonNull public TableState getTableState(TableName tableName) throws IOException { ReadWriteLock lock = tnLock.getLock(tableName); @@ -163,7 +212,7 @@ public TableState getTableState(TableName tableName) throws IOException { try { TableState currentState = readMetaState(tableName); if (currentState == null) { - throw new TableNotFoundException("No state found for " + tableName); + throw new TableStateNotFoundException(tableName); } return currentState; } finally { @@ -172,18 +221,22 @@ public TableState getTableState(TableName tableName) throws IOException { } private void updateMetaState(TableName tableName, TableState.State newState) throws IOException { + if (tableName.equals(TableName.META_TABLE_NAME)) { + if (TableState.State.DISABLING.equals(newState) || + TableState.State.DISABLED.equals(newState)) { + throw new IllegalArgumentIOException("Cannot disable the meta table; " + newState); + } + // Otherwise, just return; no need to set ENABLED on meta -- it is always ENABLED. + return; + } boolean succ = false; try { - if (tableName.equals(TableName.META_TABLE_NAME)) { - this.metaTableState = newState; - } else { - MetaTableAccessor.updateTableState(master.getConnection(), tableName, newState); - } - this.tableName2State.put(tableName, newState); + MetaTableAccessor.updateTableState(master.getConnection(), tableName, newState); + tableName2State.put(tableName, newState); succ = true; } finally { if (!succ) { - this.tableName2State.remove(tableName); + tableName2State.remove(tableName); } } metaStateUpdated(tableName, newState); @@ -202,9 +255,7 @@ private TableState readMetaState(TableName tableName) throws IOException { if (state != null) { return new TableState(tableName, state); } - TableState tableState = tableName.equals(TableName.META_TABLE_NAME)? - new TableState(TableName.META_TABLE_NAME, this.metaTableState): - MetaTableAccessor.getTableState(master.getConnection(), tableName); + TableState tableState = MetaTableAccessor.getTableState(master.getConnection(), tableName); if (tableState != null) { tableName2State.putIfAbsent(tableName, tableState.getState()); } @@ -212,8 +263,10 @@ private TableState readMetaState(TableName tableName) throws IOException { } public void start() throws IOException { + TableDescriptors tableDescriptors = master.getTableDescriptors(); migrateZooKeeper(); - fixTableStates(master.getTableDescriptors(), master.getConnection()); + Connection connection = master.getConnection(); + fixTableStates(tableDescriptors, connection); } private void fixTableStates(TableDescriptors tableDescriptors, Connection connection) @@ -282,7 +335,7 @@ private void migrateZooKeeper() throws IOException { TableState ts = null; try { ts = getTableState(entry.getKey()); - } catch (TableNotFoundException e) { + } catch (TableStateNotFoundException e) { // This can happen; table exists but no TableState. } if (ts == null) { @@ -324,4 +377,4 @@ protected void deleteZooKeeper(TableName tableName) { LOG.warn("Failed deleting table state from zookeeper", e); } } -} \ No newline at end of file +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java index e4e4803a0f0e..69bc8f70aa56 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java @@ -147,7 +147,8 @@ private void visitMetaEntry(final RegionStateVisitor visitor, final Result resul } } - void updateRegionLocation(RegionStateNode regionStateNode) throws IOException { + public void updateRegionLocation(RegionStateNode regionStateNode) + throws IOException { if (regionStateNode.getRegionInfo().isMetaRegion()) { updateMetaLocation(regionStateNode.getRegionInfo(), regionStateNode.getRegionLocation(), regionStateNode.getState()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 7b7df9892c74..34fde27d03c0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -78,7 +78,9 @@ public CreateTableProcedure(final MasterProcedureEnv env, @Override protected Flow executeFromState(final MasterProcedureEnv env, final CreateTableState state) throws InterruptedException { - LOG.info("{} execute state={}", this, state); + if (LOG.isTraceEnabled()) { + LOG.trace(this + " execute state=" + state); + } try { switch (state) { case CREATE_TABLE_PRE_OPERATION: diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java index 29c9fc7f29c2..18c194f3287a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.client.BufferedMutator; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.constraint.ConstraintException; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.master.TableStateManager; @@ -108,8 +109,8 @@ protected Flow executeFromState(final MasterProcedureEnv env, final DisableTable setNextState(DisableTableState.DISABLE_TABLE_ADD_REPLICATION_BARRIER); break; case DISABLE_TABLE_ADD_REPLICATION_BARRIER: - if (env.getMasterServices().getTableDescriptors().get(tableName). - hasGlobalReplicationScope()) { + if (env.getMasterServices().getTableDescriptors().get(tableName) + .hasGlobalReplicationScope()) { MasterFileSystem fs = env.getMasterFileSystem(); try (BufferedMutator mutator = env.getMasterServices().getConnection() .getBufferedMutator(TableName.META_TABLE_NAME)) { @@ -241,7 +242,10 @@ public TableOperationType getTableOperationType() { */ private boolean prepareDisable(final MasterProcedureEnv env) throws IOException { boolean canTableBeDisabled = true; - if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) { + if (tableName.equals(TableName.META_TABLE_NAME)) { + setFailure("master-disable-table", new ConstraintException("Cannot disable catalog table")); + canTableBeDisabled = false; + } else if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) { setFailure("master-disable-table", new TableNotFoundException(tableName)); canTableBeDisabled = false; } else if (!skipTableStateCheck) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java index 20df53dfd795..06d6a2cfc525 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,19 +20,18 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutionException; - +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.client.AsyncClusterConnection; -import org.apache.hadoop.hbase.client.AsyncConnection; -import org.apache.hadoop.hbase.client.AsyncTable; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; @@ -46,7 +45,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.EnableTableState; - @InterfaceAudience.Private public class EnableTableProcedure extends AbstractStateMachineTableProcedure { @@ -59,8 +57,7 @@ public EnableTableProcedure() { /** * Constructor - * - * @param env MasterProcedureEnv + * @param env MasterProcedureEnv * @param tableName the table to operate on */ public EnableTableProcedure(MasterProcedureEnv env, TableName tableName) { @@ -69,12 +66,11 @@ public EnableTableProcedure(MasterProcedureEnv env, TableName tableName) { /** * Constructor - * - * @param env MasterProcedureEnv + * @param env MasterProcedureEnv * @param tableName the table to operate on */ public EnableTableProcedure(MasterProcedureEnv env, TableName tableName, - ProcedurePrepareLatch syncLatch) { + ProcedurePrepareLatch syncLatch) { super(env, syncLatch); this.tableName = tableName; } @@ -103,53 +99,66 @@ protected Flow executeFromState(final MasterProcedureEnv env, final EnableTableS setNextState(EnableTableState.ENABLE_TABLE_MARK_REGIONS_ONLINE); break; case ENABLE_TABLE_MARK_REGIONS_ONLINE: - // Get the region replica count. If changed since disable, need to do - // more work assigning. - AsyncClusterConnection connection = env.getMasterServices().getAsyncClusterConnection(); - TableDescriptor tableDescriptor = + Connection connection = env.getMasterServices().getConnection(); + // we will need to get the tableDescriptor here to see if there is a change in the replica + // count + TableDescriptor hTableDescriptor = env.getMasterServices().getTableDescriptors().get(tableName); - int configuredReplicaCount = tableDescriptor.getRegionReplication(); - // Get regions for the table from memory; get both online and offline regions ('true'). + + // Get the replica count + int regionReplicaCount = hTableDescriptor.getRegionReplication(); + + // Get the regions for the table from memory; get both online and offline regions + // ('true'). List regionsOfTable = env.getAssignmentManager().getRegionStates().getRegionsOfTable(tableName, true); - // How many replicas do we currently have? Check regions returned from - // in-memory state. - int currentMaxReplica = getMaxReplicaId(regionsOfTable); - - // Read the META table to know the number of replicas the table currently has. - // If there was a table modification on region replica count then need to - // adjust replica counts here. - int replicasFound = getReplicaCount(connection, this.tableName); - LOG.info("replicasFound={} (configuredReplicaCount={} for {}", replicasFound, - configuredReplicaCount, tableName.getNameAsString()); - if (currentMaxReplica == (configuredReplicaCount - 1)) { + int currentMaxReplica = 0; + // Check if the regions in memory have replica regions as marked in META table + for (RegionInfo regionInfo : regionsOfTable) { + if (regionInfo.getReplicaId() > currentMaxReplica) { + // Iterating through all the list to identify the highest replicaID region. + // We can stop after checking with the first set of regions?? + currentMaxReplica = regionInfo.getReplicaId(); + } + } + + // read the META table to know the actual number of replicas for the table - if there + // was a table modification on region replica then this will reflect the new entries also + int replicasFound = + getNumberOfReplicasFromMeta(connection, regionReplicaCount, regionsOfTable); + assert regionReplicaCount - 1 == replicasFound; + LOG.info(replicasFound + " META entries added for the given regionReplicaCount " + + regionReplicaCount + " for the table " + tableName.getNameAsString()); + if (currentMaxReplica == (regionReplicaCount - 1)) { if (LOG.isDebugEnabled()) { - LOG.debug("No change in number of region replicas (configuredReplicaCount={});" - + " assigning.", configuredReplicaCount); + LOG.debug("There is no change to the number of region replicas." + + " Assigning the available regions." + " Current and previous" + + "replica count is " + regionReplicaCount); } - } else if (currentMaxReplica > (configuredReplicaCount - 1)) { - // We have additional regions as the replica count has been decreased. Delete + } else if (currentMaxReplica > (regionReplicaCount - 1)) { + // we have additional regions as the replica count has been decreased. Delete // those regions because already the table is in the unassigned state LOG.info("The number of replicas " + (currentMaxReplica + 1) - + " is more than the region replica count " + configuredReplicaCount); - List copyOfRegions = new ArrayList<>(regionsOfTable); + + " is more than the region replica count " + regionReplicaCount); + List copyOfRegions = new ArrayList(regionsOfTable); for (RegionInfo regionInfo : copyOfRegions) { - if (regionInfo.getReplicaId() > (configuredReplicaCount - 1)) { + if (regionInfo.getReplicaId() > (regionReplicaCount - 1)) { // delete the region from the regionStates env.getAssignmentManager().getRegionStates().deleteRegion(regionInfo); // remove it from the list of regions of the table - LOG.info("Removed replica={} of {}", regionInfo.getRegionId(), regionInfo); + LOG.info("The regioninfo being removed is " + regionInfo + " " + + regionInfo.getReplicaId()); regionsOfTable.remove(regionInfo); } } } else { // the replicasFound is less than the regionReplication - LOG.info("Number of replicas has increased. Assigning new region replicas." + - "The previous replica count was {}. The current replica count is {}.", - (currentMaxReplica + 1), configuredReplicaCount); - regionsOfTable = RegionReplicaUtil.addReplicas(tableDescriptor, regionsOfTable, - currentMaxReplica + 1, configuredReplicaCount); + LOG.info("The number of replicas has been changed(increased)." + + " Lets assign the new region replicas. The previous replica count was " + + (currentMaxReplica + 1) + ". The current replica count is " + regionReplicaCount); + regionsOfTable = RegionReplicaUtil.addReplicas(hTableDescriptor, regionsOfTable, + currentMaxReplica + 1, regionReplicaCount); } // Assign all the table regions. (including region replicas if added). // createAssignProcedure will try to retain old assignments if possible. @@ -166,32 +175,40 @@ protected Flow executeFromState(final MasterProcedureEnv env, final EnableTableS default: throw new UnsupportedOperationException("unhandled state=" + state); } - } catch (IOException | ExecutionException e) { + } catch (IOException e) { if (isRollbackSupported(state)) { - setFailure("master-enable-table", getCause(e)); + setFailure("master-enable-table", e); } else { - LOG.warn("Retryable error enabling {}, state={}", tableName, state, getCause(e)); + LOG.warn( + "Retriable error trying to enable table=" + tableName + " (in state=" + state + ")", e); } } return Flow.HAS_MORE_STATE; } - /** - * @return If ExecutionException, pull out the cause. - */ - private Throwable getCause(Exception e) { - return e instanceof ExecutionException? ((ExecutionException)e).getCause(): e; + private int getNumberOfReplicasFromMeta(Connection connection, int regionReplicaCount, + List regionsOfTable) throws IOException { + Result r = getRegionFromMeta(connection, regionsOfTable); + int replicasFound = 0; + for (int i = 1; i < regionReplicaCount; i++) { + // Since we have already added the entries to the META we will be getting only that here + List columnCells = + r.getColumnCells(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i)); + if (!columnCells.isEmpty()) { + replicasFound++; + } + } + return replicasFound; } - /** - * @return If hbase;meta table, it goes to the registry implementation which is what we want. - */ - private int getReplicaCount(AsyncConnection connection, TableName tableName) - throws ExecutionException, InterruptedException { - AsyncTable t = connection.getTable(TableName.META_TABLE_NAME); - List rls = - t.getRegionLocator().getRegionLocations(HConstants.EMPTY_START_ROW, true).get(); - return rls.size(); + private Result getRegionFromMeta(Connection connection, List regionsOfTable) + throws IOException { + byte[] metaKeyForRegion = MetaTableAccessor.getMetaKeyForRegion(regionsOfTable.get(0)); + Get get = new Get(metaKeyForRegion); + get.addFamily(HConstants.CATALOG_FAMILY); + Table metaTable = MetaTableAccessor.getMetaHTable(connection); + Result r = metaTable.get(get); + return r; } @Override @@ -391,19 +408,4 @@ private void runCoprocessorAction(final MasterProcedureEnv env, final EnableTabl } } } - - /** - * @return Maximum region replica id found in passed list of regions. - */ - private static int getMaxReplicaId(List regions) { - int max = 0; - for (RegionInfo regionInfo: regions) { - if (regionInfo.getReplicaId() > max) { - // Iterating through all the list to identify the highest replicaID region. - // We can stop after checking with the first set of regions?? - max = regionInfo.getReplicaId(); - } - } - return max; - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java index 6ab0077dc67a..755e0a3dc580 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerProcedure.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -23,10 +23,10 @@ import java.util.Map; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.master.TableStateManager; +import org.apache.hadoop.hbase.master.TableStateManager.TableStateNotFoundException; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface; import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; @@ -140,7 +140,7 @@ private boolean needSetLastPushedSequenceId(TableStateManager tsm, TableName tn) return true; } Thread.sleep(SLEEP_INTERVAL_MS); - } catch (TableNotFoundException e) { + } catch (TableStateNotFoundException e) { return false; } catch (InterruptedException e) { throw (IOException) new InterruptedIOException(e.getMessage()).initCause(e); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java index d739b6d105f7..c4df6131f376 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,10 +20,10 @@ import java.io.IOException; import java.io.InterruptedIOException; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.master.TableStateManager; +import org.apache.hadoop.hbase.master.TableStateManager.TableStateNotFoundException; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; import org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure; @@ -125,7 +125,7 @@ private boolean needReopen(TableStateManager tsm, TableName tn) throws IOExcepti return false; } Thread.sleep(SLEEP_INTERVAL_MS); - } catch (TableNotFoundException e) { + } catch (TableStateNotFoundException e) { return false; } catch (InterruptedException e) { throw (IOException) new InterruptedIOException(e.getMessage()).initCause(e); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/zksyncer/MetaLocationSyncer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/zksyncer/MetaLocationSyncer.java index 98d73224ce9b..eb80a2a232b2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/zksyncer/MetaLocationSyncer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/zksyncer/MetaLocationSyncer.java @@ -1,4 +1,4 @@ -/* +/** * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -41,6 +41,6 @@ boolean validate(String path) { @Override Collection getNodesToWatch() { - return watcher.getZNodePaths().getMetaReplicaZNodes(); + return watcher.getZNodePaths().metaReplicaZNodes.values(); } -} \ No newline at end of file +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index ebe8506e5bc1..a0a6b4cbc5f1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -744,14 +744,9 @@ private void initializeFileSystem() throws IOException { protected TableDescriptors getFsTableDescriptors() throws IOException { return new FSTableDescriptors(this.conf, - this.fs, this.rootDir, !canUpdateTableDescriptor(), false); + this.fs, this.rootDir, !canUpdateTableDescriptor(), false, getMetaTableObserver()); } - /** - * @deprecated Since 2.3.0. Not needed anymore. Was used by Master to pass in replication - * setting on hbase:meta construction. To be removed in hbase4. - */ - @Deprecated protected Function getMetaTableObserver() { return null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RejectReplicationRequestStateChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RejectReplicationRequestStateChecker.java index 92021330f4fb..9ad0af2286e4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RejectReplicationRequestStateChecker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RejectReplicationRequestStateChecker.java @@ -1,6 +1,4 @@ -/* - * Copyright The Apache Software Foundation - * +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,7 +22,6 @@ import org.apache.hadoop.hbase.replication.SyncReplicationState; import org.apache.yetus.audience.InterfaceAudience; - /** * Check whether we need to reject the replication request from source cluster. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java index 3092e2fe03cb..8ae5b4b8ddd6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,6 +24,7 @@ import java.util.Map; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -98,7 +99,10 @@ public class FSTableDescriptors implements TableDescriptors { // TODO. private final Map cache = new ConcurrentHashMap<>(); - private final Configuration configuration; + /** + * Table descriptor for hbase:meta catalog table + */ + private final TableDescriptor metaTableDescriptor; /** * Construct a FSTableDescriptors instance using the hbase root dir of the given @@ -119,22 +123,30 @@ public FSTableDescriptors(final Configuration conf, final FileSystem fs, final P * operations; i.e. on remove, we do not do delete in fs. */ public FSTableDescriptors(final Configuration conf, final FileSystem fs, - final Path rootdir, final boolean fsreadonly, final boolean usecache) throws IOException { + final Path rootdir, final boolean fsreadonly, final boolean usecache) throws IOException { + this(conf, fs, rootdir, fsreadonly, usecache, null); + } + + /** + * @param fsreadonly True if we are read-only when it comes to filesystem + * operations; i.e. on remove, we do not do delete in fs. + * @param metaObserver Used by HMaster. It need to modify the META_REPLICAS_NUM for meta table descriptor. + * see HMaster#finishActiveMasterInitialization + * TODO: This is a workaround. Should remove this ugly code... + */ + public FSTableDescriptors(final Configuration conf, final FileSystem fs, + final Path rootdir, final boolean fsreadonly, final boolean usecache, + Function metaObserver) throws IOException { this.fs = fs; this.rootdir = rootdir; this.fsreadonly = fsreadonly; this.usecache = usecache; - this.configuration = conf; + this.metaTableDescriptor = metaObserver == null ? createMetaTableDescriptor(conf) + : metaObserver.apply(createMetaTableDescriptorBuilder(conf)).build(); } - /** - * Should be private - * @deprecated Since 2.3.0. Should be for internal use only. Used by testing. - */ - @Deprecated @VisibleForTesting - public static TableDescriptorBuilder createMetaTableDescriptorBuilder(final Configuration conf) - throws IOException { + public static TableDescriptorBuilder createMetaTableDescriptorBuilder(final Configuration conf) throws IOException { // TODO We used to set CacheDataInL1 for META table. When we have BucketCache in file mode, now // the META table data goes to File mode BC only. Test how that affect the system. If too much, // we have to rethink about adding back the setCacheDataInL1 for META table CFs. @@ -179,10 +191,7 @@ public static TableDescriptorBuilder createMetaTableDescriptorBuilder(final Conf .build()) .setCoprocessor(CoprocessorDescriptorBuilder.newBuilder( MultiRowMutationEndpoint.class.getName()) - .setPriority(Coprocessor.PRIORITY_SYSTEM) - .build()) - .setRegionReplication(conf.getInt(HConstants.META_REPLICAS_NUM, - HConstants.DEFAULT_META_REPLICA_NUM)); + .setPriority(Coprocessor.PRIORITY_SYSTEM).build()); } @VisibleForTesting @@ -219,11 +228,16 @@ public boolean isUsecache() { public TableDescriptor get(final TableName tablename) throws IOException { invocations++; - // If some one tries to get the descriptor for + if (TableName.META_TABLE_NAME.equals(tablename)) { + cachehits++; + return metaTableDescriptor; + } + // hbase:meta is already handled. If some one tries to get the descriptor for // .logs, .oldlogs or .corrupt throw an exception. if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(tablename.getNameAsString())) { - throw new IOException("No descriptor found for non table = " + tablename); + throw new IOException("No descriptor found for non table = " + tablename); } + if (usecache) { // Look in cache of descriptors. TableDescriptor cachedtdm = this.cache.get(tablename); @@ -232,27 +246,24 @@ public TableDescriptor get(final TableName tablename) return cachedtdm; } } - TableDescriptor td = null; + TableDescriptor tdmt = null; try { - td = getTableDescriptorFromFs(fs, rootdir, tablename); + tdmt = getTableDescriptorFromFs(fs, rootdir, tablename); } catch (NullPointerException e) { - LOG.debug("Exception during readTableDecriptor; tableName={}", tablename, e); + LOG.debug("Exception during readTableDecriptor. Current table name = " + + tablename, e); } catch (TableInfoMissingException e) { - if (TableName.isMetaTableName(tablename)) { - // If we tried to access hbase:meta and it not there, create it. - td = createMetaTableDescriptor(this.configuration); - LOG.info("Creating new hbase:meta table default descriptor/schema {}", td); - } + // ignore. This is regular operation } catch (IOException ioe) { LOG.debug("Exception during readTableDecriptor. Current table name = " + tablename, ioe); } // last HTD written wins - if (usecache && td != null) { - this.cache.put(tablename, td); + if (usecache && tdmt != null) { + this.cache.put(tablename, tdmt); } - return td; + return tdmt; } /** @@ -262,21 +273,16 @@ public TableDescriptor get(final TableName tablename) public Map getAll() throws IOException { Map tds = new TreeMap<>(); + if (fsvisited && usecache) { - if (this.cache.get(TableName.META_TABLE_NAME) == null) { - // This get will create hbase:meta if it does not exist. Will also populate cache. - get(TableName.META_TABLE_NAME); - } for (Map.Entry entry: this.cache.entrySet()) { tds.put(entry.getKey().getNameWithNamespaceInclAsString(), entry.getValue()); } + // add hbase:meta to the response + tds.put(this.metaTableDescriptor.getTableName().getNameAsString(), metaTableDescriptor); } else { LOG.trace("Fetching table descriptors from the filesystem."); boolean allvisited = true; - // Add hbase:meta descriptor. The get will create hbase:meta in fs if doesn't - // exist. FSUtils listing table names in fs skip meta dirs. TODO: Fill out - // FSUtils with methods to get userspace tables and system tables. - tds.put(TableName.META_TABLE_NAME.toString(), get(TableName.META_TABLE_NAME)); for (Path d : FSUtils.getTableDirs(fs, rootdir)) { TableDescriptor htd = null; try { @@ -330,9 +336,14 @@ public void add(TableDescriptor htd) throws IOException { if (fsreadonly) { throw new NotImplementedException("Cannot add a table descriptor - in read only mode"); } - if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(htd.getTableName().getNameAsString())) { - throw new NotImplementedException("Cannot add Descriptor for reserved subdirectory name: " + - htd.getTableName().getNameAsString()); + TableName tableName = htd.getTableName(); + if (TableName.META_TABLE_NAME.equals(tableName)) { + throw new NotImplementedException(HConstants.NOT_IMPLEMENTED); + } + if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(tableName.getNameAsString())) { + throw new NotImplementedException( + "Cannot add a table descriptor for a reserved subdirectory name: " + + htd.getTableName().getNameAsString()); } updateTableDescriptor(htd); } @@ -358,6 +369,26 @@ public TableDescriptor remove(final TableName tablename) return descriptor; } + /** + * Checks if a current table info file exists for the given table + * + * @param tableName name of table + * @return true if exists + * @throws IOException + */ + public boolean isTableInfoExists(TableName tableName) throws IOException { + return getTableInfoPath(tableName) != null; + } + + /** + * Find the most current table info file for the given table in the hbase root directory. + * @return The file status of the current table info file or null if it does not exist + */ + private FileStatus getTableInfoPath(final TableName tableName) throws IOException { + Path tableDir = getTableDir(tableName); + return getTableInfoPath(tableDir); + } + private FileStatus getTableInfoPath(Path tableDir) throws IOException { return getTableInfoPath(fs, tableDir, !fsreadonly); @@ -372,6 +403,7 @@ private FileStatus getTableInfoPath(Path tableDir) * were sequence numbers). * * @return The file status of the current table info file or null if it does not exist + * @throws IOException */ public static FileStatus getTableInfoPath(FileSystem fs, Path tableDir) throws IOException { @@ -389,6 +421,7 @@ public static FileStatus getTableInfoPath(FileSystem fs, Path tableDir) * older files. * * @return The file status of the current table info file or null if none exist + * @throws IOException */ private static FileStatus getTableInfoPath(FileSystem fs, Path tableDir, boolean removeOldFiles) throws IOException { @@ -576,6 +609,21 @@ private static TableDescriptor readTableDescriptor(FileSystem fs, FileStatus sta return p; } + /** + * Deletes all the table descriptor files from the file system. + * Used in unit tests only. + * @throws NotImplementedException if in read only mode + */ + public void deleteTableDescriptorIfExists(TableName tableName) throws IOException { + if (fsreadonly) { + throw new NotImplementedException("Cannot delete a table descriptor - in read only mode"); + } + + Path tableDir = getTableDir(tableName); + Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR); + deleteTableDescriptorFiles(fs, tableInfoDir, Integer.MAX_VALUE); + } + /** * Deletes files matching the table info file pattern within the given directory * whose sequenceId is at most the given max sequenceId. @@ -712,27 +760,6 @@ public boolean createTableDescriptorForTableDirectory(Path tableDir, if (fsreadonly) { throw new NotImplementedException("Cannot create a table descriptor - in read only mode"); } - return createTableDescriptorForTableDirectory(this.fs, tableDir, htd, forceCreation); - } - - /** - * Create a new TableDescriptor in the specified table directory and filesystem. Happens when we - * create a new table or snapshot a table. This method doesn't require creationg of an - * {@link FSTableDescriptors} instance so it takes a bunch of arguments. Users of the method - * above used to create an FSTableDescriptors instance just to run the method. That was fine - * until construction started expecting to be able to read the hbase:meta schema. Snapshotting - * to some random dir would fail construction if no hbase:meta schema available. - * @param fs Filesystem to write to. Snapshot can set it to other than that of running system. - * @param tableDir table directory under which we should write the file - * @param htd description of the table to write - * @param forceCreation if true,then even if previous table descriptor is present it will - * be overwritten - * @return true if the we successfully created the file, false if the file - * already exists and we weren't forcing the descriptor creation. - * @throws IOException if a filesystem error occurs - */ - public static boolean createTableDescriptorForTableDirectory(FileSystem fs, Path tableDir, - TableDescriptor htd, boolean forceCreation) throws IOException { FileStatus status = getTableInfoPath(fs, tableDir); if (status != null) { LOG.debug("Current path=" + status.getPath()); @@ -748,5 +775,6 @@ public static boolean createTableDescriptorForTableDirectory(FileSystem fs, Path Path p = writeTableDescriptor(fs, htd, tableDir, status); return p != null; } + } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 54c5c17bf675..c5f6d32096f5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -121,6 +121,7 @@ import org.apache.hadoop.hbase.trace.TraceUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; @@ -156,7 +157,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; - /** * Facility for testing HBase. Replacement for * old HBaseTestCase and HBaseClusterTestCase functionality. @@ -494,6 +494,17 @@ private Path getBaseTestDirOnTestFS() throws IOException { return new Path(fs.getWorkingDirectory(), "test-data"); } + /** + * @return META table descriptor + */ + public TableDescriptorBuilder getMetaTableDescriptorBuilder() { + try { + return FSTableDescriptors.createMetaTableDescriptorBuilder(conf); + } catch (IOException e) { + throw new RuntimeException("Unable to create META table descriptor", e); + } + } + /** * Returns a Path in the test filesystem, obtained from {@link #getTestFileSystem()} * to write temporary test data. Call this method after setting up the mini dfs cluster diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseMetaEdit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseMetaEdit.java deleted file mode 100644 index d201ed06ec38..000000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseMetaEdit.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; - -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.RegionInfoBuilder; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; -import org.apache.hadoop.hbase.regionserver.Region; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; - - -/** - * Test being able to edit hbase:meta. - */ -@Category({MiscTests.class, LargeTests.class}) -public class TestHBaseMetaEdit { - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestHBaseMetaEdit.class); - @Rule - public TestName name = new TestName(); - private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); - - @Before - public void before() throws Exception { - UTIL.startMiniCluster(); - } - - @After - public void after() throws Exception { - UTIL.shutdownMiniCluster(); - } - - /** - * Set versions, set HBASE-16213 indexed block encoding, and add a column family. - * Verify they are all in place by looking at TableDescriptor AND by checking - * what the RegionServer sees after opening Region. - */ - @Test - public void testEditMeta() throws IOException { - Admin admin = UTIL.getAdmin(); - admin.disableTable(TableName.META_TABLE_NAME); - TableDescriptor descriptor = admin.getDescriptor(TableName.META_TABLE_NAME); - ColumnFamilyDescriptor cfd = descriptor.getColumnFamily(HConstants.CATALOG_FAMILY); - byte [] extraColumnFamilyName = Bytes.toBytes("xtra"); - ColumnFamilyDescriptor newCfd = - ColumnFamilyDescriptorBuilder.newBuilder(extraColumnFamilyName).build(); - int oldVersions = cfd.getMaxVersions(); - // Add '1' to current versions count. - cfd = ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(oldVersions + 1). - setConfiguration(ColumnFamilyDescriptorBuilder.DATA_BLOCK_ENCODING, - DataBlockEncoding.ROW_INDEX_V1.toString()).build(); - admin.modifyColumnFamily(TableName.META_TABLE_NAME, cfd); - admin.addColumnFamily(TableName.META_TABLE_NAME, newCfd); - descriptor = admin.getDescriptor(TableName.META_TABLE_NAME); - // Assert new max versions is == old versions plus 1. - assertEquals(oldVersions + 1, - descriptor.getColumnFamily(HConstants.CATALOG_FAMILY).getMaxVersions()); - admin.enableTable(TableName.META_TABLE_NAME); - descriptor = admin.getDescriptor(TableName.META_TABLE_NAME); - // Assert new max versions is == old versions plus 1. - assertEquals(oldVersions + 1, - descriptor.getColumnFamily(HConstants.CATALOG_FAMILY).getMaxVersions()); - assertTrue(descriptor.getColumnFamily(newCfd.getName()) != null); - String encoding = descriptor.getColumnFamily(HConstants.CATALOG_FAMILY).getConfiguration(). - get(ColumnFamilyDescriptorBuilder.DATA_BLOCK_ENCODING); - assertEquals(encoding, DataBlockEncoding.ROW_INDEX_V1.toString()); - Region r = UTIL.getHBaseCluster().getRegionServer(0). - getRegion(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); - assertEquals(oldVersions + 1, - r.getStore(HConstants.CATALOG_FAMILY).getColumnFamilyDescriptor().getMaxVersions()); - encoding = r.getStore(HConstants.CATALOG_FAMILY).getColumnFamilyDescriptor(). - getConfigurationValue(ColumnFamilyDescriptorBuilder.DATA_BLOCK_ENCODING); - assertEquals(encoding, DataBlockEncoding.ROW_INDEX_V1.toString()); - assertTrue(r.getStore(extraColumnFamilyName) != null); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java index d723db192262..55b85bb1fc32 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.Waiter.Predicate; +import org.apache.hadoop.hbase.constraint.ConstraintException; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -503,6 +504,22 @@ private HRegionServer startAndWriteData(TableName tableName, byte[] value) return regionServer; } + @Test + public void testDisableCatalogTable() throws Exception { + try { + ADMIN.disableTable(TableName.META_TABLE_NAME); + fail("Expected to throw ConstraintException"); + } catch (ConstraintException e) { + } + // Before the fix for HBASE-6146, the below table creation was failing as the hbase:meta table + // actually getting disabled by the disableTable() call. + HTableDescriptor htd = + new HTableDescriptor(TableName.valueOf(Bytes.toBytes(name.getMethodName()))); + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("cf1")); + htd.addFamily(hcd); + TEST_UTIL.getAdmin().createTable(htd); + } + @Test public void testIsEnabledOrDisabledOnUnknownTable() throws Exception { try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAccessControlAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAccessControlAdminApi.java index 33c8c109cc7c..9182e6fb9fb2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAccessControlAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAccessControlAdminApi.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one or more contributor license * agreements. See the NOTICE file distributed with this work for additional information regarding * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi2.java index cd20e1038eb5..4f5cbf219d6c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi2.java @@ -39,6 +39,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Class to test asynchronous table admin operations @@ -53,6 +54,18 @@ public class TestAsyncTableAdminApi2 extends TestAsyncAdminBase { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestAsyncTableAdminApi2.class); + @Test + public void testDisableCatalogTable() throws Exception { + try { + this.admin.disableTable(TableName.META_TABLE_NAME).join(); + fail("Expected to throw ConstraintException"); + } catch (Exception e) { + } + // Before the fix for HBASE-6146, the below table creation was failing as the hbase:meta table + // actually getting disabled by the disableTable() call. + createTableWithDefaultConf(tableName); + } + @Test public void testAddColumnFamily() throws Exception { // Create a table with two families diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java index c0b211b5ebac..d4bb3be02044 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi3.java @@ -18,9 +18,12 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; +import static org.hamcrest.CoreMatchers.instanceOf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.ArrayList; import java.util.Collections; @@ -29,6 +32,7 @@ import java.util.concurrent.ExecutionException; import java.util.regex.Pattern; import org.apache.hadoop.hbase.AsyncMetaTableAccessor; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; @@ -197,6 +201,14 @@ public void testDisableAndEnableTable() throws Exception { ok = false; } assertTrue(ok); + // meta table can not be disabled. + try { + admin.disableTable(TableName.META_TABLE_NAME).get(); + fail("meta table can not be disabled"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertThat(cause, instanceOf(DoNotRetryIOException.class)); + } } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java index 9b533c0f9eb4..55fc289dca91 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java @@ -147,13 +147,6 @@ public void tearDown() throws Exception { public void testMetaHTDReplicaCount() throws Exception { assertEquals(3, TEST_UTIL.getAdmin().getDescriptor(TableName.META_TABLE_NAME).getRegionReplication()); - try (AsyncConnection connection = - ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get()) { - AsyncTable t = connection.getTable(TableName.META_TABLE_NAME); - List rls = - t.getRegionLocator().getRegionLocations(HConstants.EMPTY_START_ROW, true).get(); - assertEquals(3, rls.size()); - } } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java index 128a24745a53..189fd2ea1606 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java index 6e6b3524b629..3da03416b44d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java @@ -47,7 +47,6 @@ import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.wal.WAL; import org.junit.ClassRule; import org.junit.Rule; @@ -88,10 +87,8 @@ public class TestGetClosestAtOrBefore { public void testUsingMetaAndBinary() throws IOException { FileSystem filesystem = FileSystem.get(conf); Path rootdir = UTIL.getDataTestDirOnTestFS(); - // Up flush size else we bind up when we use default catalog flush of 16k. - TableDescriptorBuilder metaBuilder = - FSTableDescriptors.createMetaTableDescriptorBuilder(UTIL.getConfiguration()) + TableDescriptorBuilder metaBuilder = UTIL.getMetaTableDescriptorBuilder() .setMemStoreFlushSize(64 * 1024 * 1024); HRegion mr = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java index 8a98d4a1d1dd..819df673c94e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL; @@ -78,6 +77,8 @@ public void sync(boolean forceSync) throws IOException { /** * Spin up a bunch of threads and have them all append to a WAL. Roll the * WAL frequently to try and trigger NPE. + * @throws IOException + * @throws InterruptedException */ @Test public void testContendedLogRolling() throws Exception { @@ -160,8 +161,7 @@ public void run() { byte[] bytes = Bytes.toBytes(i); edit.add(new KeyValue(bytes, bytes, bytes, now, EMPTY_1K_ARRAY)); RegionInfo hri = RegionInfoBuilder.FIRST_META_REGIONINFO; - TableDescriptor htd = - FSTableDescriptors.createMetaTableDescriptor(TEST_UTIL.getConfiguration()); + TableDescriptor htd = TEST_UTIL.getMetaTableDescriptorBuilder().build(); NavigableMap scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); for(byte[] fam : htd.getColumnFamilyNames()) { scopes.put(fam, 0); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java index fe68948647f5..a0c37f27d4e1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java @@ -322,9 +322,7 @@ public void testGetAllOrdering() throws Exception { } Map tables = tds.getAll(); - assertEquals(5, tables.size()); - // Remove because it messes up below order test. - tables.remove(TableName.META_TABLE_NAME.toString()); + assertEquals(4, tables.size()); String[] tableNamesOrdered = new String[] { "bar:foo", "default:bar", "default:foo", "foo:bar" }; @@ -370,9 +368,6 @@ public void testCacheConsistency() for (Map.Entry entry: nonchtds.getAll().entrySet()) { String t = (String) entry.getKey(); - if (t.equals(TableName.META_TABLE_NAME.toString())) { - continue; - } TableDescriptor nchtd = entry.getValue(); assertTrue("expected " + htd.toString() + " got: " + chtds.get(TableName.valueOf(t)).toString(), diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java index 1183cd675893..0cebc762fd73 100644 --- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java +++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -61,6 +61,14 @@ public final class MetaTableLocator { private MetaTableLocator() { } + /** + * Checks if the meta region location is available. + * @return true if meta region location is available, false if not + */ + public static boolean isLocationAvailable(ZKWatcher zkw) { + return getMetaRegionLocation(zkw) != null; + } + /** * @param zkw ZooKeeper watcher to be used * @return meta table regions and their locations. @@ -258,7 +266,7 @@ public static RegionState getMetaRegionState(ZKWatcher zkw) throws KeeperExcepti } /** - * Load the meta region state from the meta region server ZNode. + * Load the meta region state from the meta server ZNode. * * @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation * @param replicaId the ID of the replica @@ -298,8 +306,10 @@ public static RegionState getMetaRegionState(ZKWatcher zkw, int replicaId) if (serverName == null) { state = RegionState.State.OFFLINE; } - return new RegionState(RegionReplicaUtil.getRegionInfoForReplica( - RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId), state, serverName); + return new RegionState( + RegionReplicaUtil.getRegionInfoForReplica( + RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId), + state, serverName); } /** diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java index 2e3e4b03b441..878f3ca6e777 100644 --- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java +++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java @@ -2057,7 +2057,7 @@ private static void logRetrievedMsg(final ZKWatcher zkw, " byte(s) of data from znode " + znode + (watcherSet? " and set watcher; ": "; data=") + (data == null? "null": data.length == 0? "empty": ( - zkw.getZNodePaths().isMetaZNodePrefix(znode)? + znode.startsWith(zkw.getZNodePaths().metaZNodePrefix)? getServerNameOrEmptyString(data): znode.startsWith(zkw.getZNodePaths().backupMasterAddressesZNode)? getServerNameOrEmptyString(data):