diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml index f51617efdf18..a54df8cc0290 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml @@ -333,20 +333,4 @@ pom - - - rsgroup - - - !skip-rsgroup - - - - - org.apache.hbase - hbase-rsgroup - - - - diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml index e4a3d36700b1..4238da5bfbc0 100644 --- a/hbase-assembly/src/main/assembly/components.xml +++ b/hbase-assembly/src/main/assembly/components.xml @@ -144,14 +144,6 @@ 0644 - - ${project.basedir}/../hbase-rsgroup/target/ - lib - - ${rsgroup.test.jar} - - 0644 - ${project.basedir}/../hbase-mapreduce/target/ lib diff --git a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml index fcfb442c3b11..f2dae4d31adc 100644 --- a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml +++ b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml @@ -50,7 +50,6 @@ org.apache.hbase:hbase-protocol-shaded org.apache.hbase:hbase-replication org.apache.hbase:hbase-rest - org.apache.hbase:hbase-rsgroup org.apache.hbase:hbase-server org.apache.hbase:hbase-shell org.apache.hbase:hbase-testing-util diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index e69036ae79cf..ca4e0696d15f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -97,6 +97,7 @@ import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos; import org.apache.hadoop.hbase.protobuf.generated.TableProtos; import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; +import org.apache.hadoop.hbase.protobuf.generated.TableProtos; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.util.Addressing; @@ -1775,6 +1776,14 @@ public static ServerName toServerName(final byte [] data) throws Deserialization return ServerName.valueOf(hostname, port, -1L); } + public static HBaseProtos.TimeRange toTimeRange(TimeRange timeRange) { + if (timeRange == null) { + timeRange = TimeRange.allTime(); + } + return HBaseProtos.TimeRange.newBuilder().setFrom(timeRange.getMin()).setTo(timeRange.getMax()) + .build(); + } + public static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) { RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName()); for(HBaseProtos.ServerName el: proto.getServersList()) { @@ -1786,13 +1795,18 @@ public static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) { return RSGroupInfo; } - public static HBaseProtos.TimeRange toTimeRange(TimeRange timeRange) { - if (timeRange == null) { - timeRange = TimeRange.allTime(); + public static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) { + List tables = new ArrayList<>(pojo.getTables().size()); + for (TableName arg : pojo.getTables()) { + tables.add(ProtobufUtil.toProtoTableName(arg)); + } + List hostports = new ArrayList<>(pojo.getServers().size()); + for (Address el : pojo.getServers()) { + hostports.add(HBaseProtos.ServerName.newBuilder().setHostName(el.getHostname()) + .setPort(el.getPort()).build()); } - return HBaseProtos.TimeRange.newBuilder().setFrom(timeRange.getMin()) - .setTo(timeRange.getMax()) - .build(); + return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName()).addAllServers(hostports) + .addAllTables(tables).build(); } /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index 72d728af9916..0618c0356447 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -43,7 +43,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.apache.commons.io.input.BoundedInputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ByteBufferExtendedCell; @@ -94,6 +93,7 @@ import org.apache.hadoop.hbase.filter.ByteArrayComparable; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.protobuf.ProtobufMagic; import org.apache.hadoop.hbase.protobuf.ProtobufMessageConverter; import org.apache.hadoop.hbase.quotas.QuotaScope; @@ -102,6 +102,7 @@ import org.apache.hadoop.hbase.quotas.ThrottleType; import org.apache.hadoop.hbase.replication.ReplicationLoadSink; import org.apache.hadoop.hbase.replication.ReplicationLoadSource; +import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.security.visibility.Authorizations; import org.apache.hadoop.hbase.security.visibility.CellVisibility; import org.apache.hadoop.hbase.util.Addressing; @@ -178,6 +179,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MajorCompactionTimestampResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; @@ -3392,4 +3394,29 @@ public static RegionStatesCount toTableRegionStatesCount( .build(); } + + public static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) { + RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName()); + for (HBaseProtos.ServerName el : proto.getServersList()) { + RSGroupInfo.addServer(Address.fromParts(el.getHostName(), el.getPort())); + } + for (HBaseProtos.TableName pTableName : proto.getTablesList()) { + RSGroupInfo.addTable(ProtobufUtil.toTableName(pTableName)); + } + return RSGroupInfo; + } + + public static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) { + List tables = new ArrayList<>(pojo.getTables().size()); + for (TableName arg : pojo.getTables()) { + tables.add(ProtobufUtil.toProtoTableName(arg)); + } + List hostports = new ArrayList<>(pojo.getServers().size()); + for (Address el : pojo.getServers()) { + hostports.add(HBaseProtos.ServerName.newBuilder().setHostName(el.getHostname()) + .setPort(el.getPort()).build()); + } + return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName()).addAllServers(hostports) + .addAllTables(tables).build(); + } } diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index a89dd56ccfff..dd6dfd5336b1 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -181,16 +181,6 @@ test-jar test - - org.apache.hbase - hbase-rsgroup - - - org.apache.hbase - hbase-rsgroup - test-jar - test - org.apache.hbase hbase-server diff --git a/hbase-rsgroup/src/test/resources/hbase-site.xml b/hbase-protocol-shaded/src/main/protobuf/RSGroup.proto similarity index 67% rename from hbase-rsgroup/src/test/resources/hbase-site.xml rename to hbase-protocol-shaded/src/main/protobuf/RSGroup.proto index 99d2ab8d1fbb..ede2b13cb5cc 100644 --- a/hbase-rsgroup/src/test/resources/hbase-site.xml +++ b/hbase-protocol-shaded/src/main/protobuf/RSGroup.proto @@ -1,8 +1,4 @@ - - - - - - hbase.defaults.for.version.skip - true - - - hbase.hconnection.threads.keepalivetime - 3 - - + +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated"; +option java_outer_classname = "RSGroupProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; + +message RSGroupInfo { + required string name = 1; + repeated ServerName servers = 4; + repeated TableName tables = 3; +} diff --git a/hbase-protocol-shaded/src/main/protobuf/RSGroupAdmin.proto b/hbase-protocol-shaded/src/main/protobuf/RSGroupAdmin.proto new file mode 100644 index 000000000000..1db713677f2c --- /dev/null +++ b/hbase-protocol-shaded/src/main/protobuf/RSGroupAdmin.proto @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated"; +option java_outer_classname = "RSGroupAdminProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "RSGroup.proto"; + +/** Group level protobufs */ + +message ListTablesOfRSGroupRequest { + required string r_s_group_name = 1; +} + +message ListTablesOfRSGroupResponse { + repeated TableName table_name = 1; +} + +message GetRSGroupInfoRequest { + required string r_s_group_name = 1; +} + +message GetRSGroupInfoResponse { + optional RSGroupInfo r_s_group_info = 1; +} + +message GetRSGroupInfoOfTableRequest { + required TableName table_name = 1; +} + +message GetRSGroupInfoOfTableResponse { + optional RSGroupInfo r_s_group_info = 1; +} + +message MoveServersRequest { + required string target_group = 1; + repeated ServerName servers = 3; +} + +message MoveServersResponse { +} + +message MoveTablesRequest { + required string target_group = 1; + repeated TableName table_name = 2; +} + +message MoveTablesResponse { +} + +message AddRSGroupRequest { + required string r_s_group_name = 1; +} + +message AddRSGroupResponse { +} + +message RemoveRSGroupRequest { + required string r_s_group_name = 1; +} + +message RemoveRSGroupResponse { +} + +message BalanceRSGroupRequest { + required string r_s_group_name = 1; +} + +message BalanceRSGroupResponse { + required bool balanceRan = 1; +} + +message ListRSGroupInfosRequest { +} + +message ListRSGroupInfosResponse { + repeated RSGroupInfo r_s_group_info = 1; +} + +message GetRSGroupInfoOfServerRequest { + required ServerName server = 2; +} + +message GetRSGroupInfoOfServerResponse { + optional RSGroupInfo r_s_group_info = 1; +} + +message MoveServersAndTablesRequest { + required string target_group = 1; + repeated ServerName servers = 2; + repeated TableName table_name = 3; +} + +message MoveServersAndTablesResponse { +} + +message RemoveServersRequest { + repeated ServerName servers = 1; +} + +message RemoveServersResponse { +} + +service RSGroupAdminService { + rpc GetRSGroupInfo(GetRSGroupInfoRequest) + returns (GetRSGroupInfoResponse); + + rpc GetRSGroupInfoOfTable(GetRSGroupInfoOfTableRequest) + returns (GetRSGroupInfoOfTableResponse); + + rpc GetRSGroupInfoOfServer(GetRSGroupInfoOfServerRequest) + returns (GetRSGroupInfoOfServerResponse); + + rpc MoveServers(MoveServersRequest) + returns (MoveServersResponse); + + rpc MoveTables(MoveTablesRequest) + returns (MoveTablesResponse); + + rpc AddRSGroup(AddRSGroupRequest) + returns (AddRSGroupResponse); + + rpc RemoveRSGroup(RemoveRSGroupRequest) + returns (RemoveRSGroupResponse); + + rpc BalanceRSGroup(BalanceRSGroupRequest) + returns (BalanceRSGroupResponse); + + rpc ListRSGroupInfos(ListRSGroupInfosRequest) + returns (ListRSGroupInfosResponse); + + rpc MoveServersAndTables(MoveServersAndTablesRequest) + returns (MoveServersAndTablesResponse); + + rpc RemoveServers(RemoveServersRequest) + returns (RemoveServersResponse); +} diff --git a/hbase-rsgroup/src/main/protobuf/RSGroupAdmin.proto b/hbase-protocol/src/main/protobuf/RSGroupAdmin.proto similarity index 100% rename from hbase-rsgroup/src/main/protobuf/RSGroupAdmin.proto rename to hbase-protocol/src/main/protobuf/RSGroupAdmin.proto diff --git a/hbase-rsgroup/README.txt b/hbase-rsgroup/README.txt deleted file mode 100644 index b24aee650603..000000000000 --- a/hbase-rsgroup/README.txt +++ /dev/null @@ -1,13 +0,0 @@ -ON PROTOBUFS -This maven module has protobuf definition files ('.protos') used by hbase -Coprocessor Endpoints that ship with hbase core including tests. Coprocessor -Endpoints are meant to be standalone, independent code not reliant on hbase -internals. They define their Service using protobuf. The protobuf version -they use can be distinct from that used by HBase internally since HBase started -shading its protobuf references. Endpoints have no access to the shaded protobuf -hbase uses. They do have access to the content of hbase-protocol -- the -.protos found in here -- but avoid using as much of this as you can as it is -liable to change. - -Generation of java files from protobuf .proto files included here is done as -part of the build. diff --git a/hbase-rsgroup/pom.xml b/hbase-rsgroup/pom.xml deleted file mode 100644 index 86676dd43a45..000000000000 --- a/hbase-rsgroup/pom.xml +++ /dev/null @@ -1,279 +0,0 @@ - - - - 4.0.0 - - hbase-build-configuration - org.apache.hbase - 2.3.0-mdh3.2.6-SNAPSHOT - ../hbase-build-configuration - - hbase-rsgroup - Apache HBase - RSGroup - Regionserver Groups for HBase - - - - - maven-assembly-plugin - - true - - - - - org.apache.maven.plugins - maven-source-plugin - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - compile-protoc - generate-sources - - compile - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - true - - - - net.revelc.code - warbucks-maven-plugin - - - - - - - org.apache.hbase - hbase-annotations - test-jar - test - - - org.apache.hbase - hbase-client - - - org.apache.hbase - hbase-server - - - org.apache.hbase - hbase-common - - - org.apache.hbase - hbase-procedure - - - org.apache.hbase - hbase-procedure - test-jar - test - - - org.apache.hbase - hbase-protocol - - - org.apache.hbase - hbase-protocol-shaded - - - org.apache.hbase - hbase-testing-util - test - - - - org.apache.commons - commons-lang3 - - - org.slf4j - slf4j-api - - - com.xiaomi.infra.thirdparty - infra-shaded-miscellaneous - - - com.google.protobuf - protobuf-java - - - org.apache.zookeeper - zookeeper - - - log4j - log4j - test - - - org.mockito - mockito-core - test - - - junit - junit - test - - - - - - skipRSGroupTests - - - skipRSGroupTests - - - - true - true - - - - - hadoop-2.0 - - - - - !hadoop.profile - - - - - com.github.stephenc.findbugs - findbugs-annotations - true - - - org.apache.hadoop - hadoop-common - - - net.java.dev.jets3t - jets3t - - - javax.servlet.jsp - jsp-api - - - org.mortbay.jetty - jetty - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - javax.servlet - servlet-api - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - com.google.code.findbugs - jsr305 - - - - - - - - hadoop-3.0 - - - hadoop.profile - 3.0 - - - - 3.0-SNAPSHOT - - - - org.apache.hadoop - hadoop-common - - - - - eclipse-specific - - - m2e.version - - - - - - - - org.eclipse.m2e - lifecycle-mapping - - - - - - - - - - - - - diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java deleted file mode 100644 index 88dd45c8b773..000000000000 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java +++ /dev/null @@ -1,538 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.rsgroup; - -import com.google.protobuf.RpcCallback; -import com.google.protobuf.RpcController; -import com.google.protobuf.Service; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.CoprocessorEnvironment; -import org.apache.hadoop.hbase.HBaseIOException; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MasterNotRunningException; -import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.PleaseHoldException; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.SnapshotDescription; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.constraint.ConstraintException; -import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor; -import org.apache.hadoop.hbase.coprocessor.HasMasterServices; -import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; -import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; -import org.apache.hadoop.hbase.coprocessor.MasterObserver; -import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; -import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.net.Address; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RSGroupAdminService; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersResponse; -import org.apache.hadoop.hbase.protobuf.generated.TableProtos; -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.AccessChecker; -import org.apache.hadoop.hbase.security.access.Permission.Action; -import org.apache.hadoop.util.Shell.ShellCommandExecutor; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import com.xiaomi.infra.thirdparty.com.google.common.collect.Sets; - -// TODO: Encapsulate MasterObserver functions into separate subclass. -@CoreCoprocessor -@InterfaceAudience.Private -public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver { - private static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminEndpoint.class); - - private MasterServices master = null; - // Only instance of RSGroupInfoManager. RSGroup aware load balancers ask for this instance on - // their setup. - private RSGroupInfoManager groupInfoManager; - private RSGroupAdminServer groupAdminServer; - private final RSGroupAdminService groupAdminService = new RSGroupAdminServiceImpl(); - private AccessChecker accessChecker; - - /** Provider for mapping principal names to Users */ - private UserProvider userProvider; - - @Override - public void start(CoprocessorEnvironment env) throws IOException { - if (!(env instanceof HasMasterServices)) { - throw new IOException("Does not implement HMasterServices"); - } - - master = ((HasMasterServices)env).getMasterServices(); - groupInfoManager = RSGroupInfoManagerImpl.getInstance(master); - groupAdminServer = new RSGroupAdminServer(master, groupInfoManager); - Class clazz = - master.getConfiguration().getClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, null); - if (!RSGroupableBalancer.class.isAssignableFrom(clazz)) { - throw new IOException("Configured balancer does not support RegionServer groups."); - } - accessChecker = ((HasMasterServices) env).getMasterServices().getAccessChecker(); - - // set the user-provider. - this.userProvider = UserProvider.instantiate(env.getConfiguration()); - } - - @Override - public void stop(CoprocessorEnvironment env) { - } - - @Override - public Iterable getServices() { - return Collections.singleton(groupAdminService); - } - - @Override - public Optional getMasterObserver() { - return Optional.of(this); - } - - RSGroupInfoManager getGroupInfoManager() { - return groupInfoManager; - } - - /** - * Implementation of RSGroupAdminService defined in RSGroupAdmin.proto. - * This class calls {@link RSGroupAdminServer} for actual work, converts result to protocol - * buffer response, handles exceptions if any occurred and then calls the {@code RpcCallback} with - * the response. - */ - private class RSGroupAdminServiceImpl extends RSGroupAdminProtos.RSGroupAdminService { - @Override - public void getRSGroupInfo(RpcController controller, - GetRSGroupInfoRequest request, RpcCallback done) { - GetRSGroupInfoResponse.Builder builder = GetRSGroupInfoResponse.newBuilder(); - String groupName = request.getRSGroupName(); - LOG.info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, group=" - + groupName); - try { - checkPermission("getRSGroupInfo"); - RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); - if (rsGroupInfo != null) { - builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(rsGroupInfo)); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void getRSGroupInfoOfTable(RpcController controller, - GetRSGroupInfoOfTableRequest request, RpcCallback done) { - GetRSGroupInfoOfTableResponse.Builder builder = GetRSGroupInfoOfTableResponse.newBuilder(); - TableName tableName = ProtobufUtil.toTableName(request.getTableName()); - LOG.info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, table=" - + tableName); - try { - checkPermission("getRSGroupInfoOfTable"); - RSGroupInfo RSGroupInfo = groupAdminServer.getRSGroupInfoOfTable(tableName); - if (RSGroupInfo != null) { - builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo)); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void moveServers(RpcController controller, MoveServersRequest request, - RpcCallback done) { - MoveServersResponse.Builder builder = MoveServersResponse.newBuilder(); - Set
hostPorts = Sets.newHashSet(); - for (HBaseProtos.ServerName el : request.getServersList()) { - hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); - } - LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts +" to rsgroup " - + request.getTargetGroup()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preMoveServers(hostPorts, request.getTargetGroup()); - } - checkPermission("moveServers"); - groupAdminServer.moveServers(hostPorts, request.getTargetGroup()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postMoveServers(hostPorts, request.getTargetGroup()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void moveTables(RpcController controller, MoveTablesRequest request, - RpcCallback done) { - MoveTablesResponse.Builder builder = MoveTablesResponse.newBuilder(); - Set tables = new HashSet<>(request.getTableNameList().size()); - for (TableProtos.TableName tableName : request.getTableNameList()) { - tables.add(ProtobufUtil.toTableName(tableName)); - } - LOG.info(master.getClientIdAuditPrefix() + " move tables " + tables +" to rsgroup " - + request.getTargetGroup()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preMoveTables(tables, request.getTargetGroup()); - } - checkPermission("moveTables"); - groupAdminServer.moveTables(tables, request.getTargetGroup()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postMoveTables(tables, request.getTargetGroup()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void addRSGroup(RpcController controller, AddRSGroupRequest request, - RpcCallback done) { - AddRSGroupResponse.Builder builder = AddRSGroupResponse.newBuilder(); - LOG.info(master.getClientIdAuditPrefix() + " add rsgroup " + request.getRSGroupName()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preAddRSGroup(request.getRSGroupName()); - } - checkPermission("addRSGroup"); - groupAdminServer.addRSGroup(request.getRSGroupName()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postAddRSGroup(request.getRSGroupName()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void removeRSGroup(RpcController controller, - RemoveRSGroupRequest request, RpcCallback done) { - RemoveRSGroupResponse.Builder builder = - RemoveRSGroupResponse.newBuilder(); - LOG.info(master.getClientIdAuditPrefix() + " remove rsgroup " + request.getRSGroupName()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preRemoveRSGroup(request.getRSGroupName()); - } - checkPermission("removeRSGroup"); - groupAdminServer.removeRSGroup(request.getRSGroupName()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postRemoveRSGroup(request.getRSGroupName()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void balanceRSGroup(RpcController controller, - BalanceRSGroupRequest request, RpcCallback done) { - BalanceRSGroupResponse.Builder builder = BalanceRSGroupResponse.newBuilder(); - LOG.info(master.getClientIdAuditPrefix() + " balance rsgroup, group=" - + request.getRSGroupName()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preBalanceRSGroup(request.getRSGroupName()); - } - checkPermission("balanceRSGroup"); - boolean balancerRan = groupAdminServer.balanceRSGroup(request.getRSGroupName()); - builder.setBalanceRan(balancerRan); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postBalanceRSGroup(request.getRSGroupName(), - balancerRan); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - builder.setBalanceRan(false); - } - done.run(builder.build()); - } - - @Override - public void listRSGroupInfos(RpcController controller, - ListRSGroupInfosRequest request, RpcCallback done) { - ListRSGroupInfosResponse.Builder builder = ListRSGroupInfosResponse.newBuilder(); - LOG.info(master.getClientIdAuditPrefix() + " list rsgroup"); - try { - checkPermission("listRSGroup"); - for (RSGroupInfo RSGroupInfo : groupAdminServer.listRSGroups()) { - builder.addRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo)); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void getRSGroupInfoOfServer(RpcController controller, - GetRSGroupInfoOfServerRequest request, RpcCallback done) { - GetRSGroupInfoOfServerResponse.Builder builder = GetRSGroupInfoOfServerResponse.newBuilder(); - Address hp = Address.fromParts(request.getServer().getHostName(), - request.getServer().getPort()); - LOG.info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, server=" - + hp); - try { - checkPermission("getRSGroupInfoOfServer"); - RSGroupInfo info = groupAdminServer.getRSGroupOfServer(hp); - if (info != null) { - builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(info)); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void moveServersAndTables(RpcController controller, - MoveServersAndTablesRequest request, RpcCallback done) { - MoveServersAndTablesResponse.Builder builder = MoveServersAndTablesResponse.newBuilder(); - Set
hostPorts = Sets.newHashSet(); - for (HBaseProtos.ServerName el : request.getServersList()) { - hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); - } - Set tables = new HashSet<>(request.getTableNameList().size()); - for (TableProtos.TableName tableName : request.getTableNameList()) { - tables.add(ProtobufUtil.toTableName(tableName)); - } - LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts - + " and tables " + tables + " to rsgroup" + request.getTargetGroup()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preMoveServersAndTables(hostPorts, tables, - request.getTargetGroup()); - } - checkPermission("moveServersAndTables"); - groupAdminServer.moveServersAndTables(hostPorts, tables, request.getTargetGroup()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postMoveServersAndTables(hostPorts, tables, - request.getTargetGroup()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void removeServers(RpcController controller, - RemoveServersRequest request, - RpcCallback done) { - RemoveServersResponse.Builder builder = - RemoveServersResponse.newBuilder(); - Set
servers = Sets.newHashSet(); - for (HBaseProtos.ServerName el : request.getServersList()) { - servers.add(Address.fromParts(el.getHostName(), el.getPort())); - } - LOG.info(master.getClientIdAuditPrefix() - + " remove decommissioned servers from rsgroup: " + servers); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preRemoveServers(servers); - } - checkPermission("removeServers"); - groupAdminServer.removeServers(servers); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postRemoveServers(servers); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - } - - boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException { - String groupName; - try { - groupName = - master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) - .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); - if (groupName == null) { - groupName = RSGroupInfo.DEFAULT_GROUP; - } - } catch (MasterNotRunningException | PleaseHoldException e) { - LOG.info("Master has not initialized yet; temporarily using default RSGroup '" + - RSGroupInfo.DEFAULT_GROUP + "' for deploy of system table"); - groupName = RSGroupInfo.DEFAULT_GROUP; - } - - RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); - if (rsGroupInfo == null) { - throw new ConstraintException( - "Default RSGroup (" + groupName + ") for this table's " + "namespace does not exist."); - } - - for (ServerName onlineServer : master.getServerManager().createDestinationServersList()) { - if (rsGroupInfo.getServers().contains(onlineServer.getAddress())) { - return true; - } - } - return false; - } - - void assignTableToGroup(TableDescriptor desc) throws IOException { - RSGroupInfo rsGroupInfo = groupInfoManager.determineRSGroupInfoForTable(desc.getTableName()); - if (rsGroupInfo == null) { - throw new ConstraintException("Default RSGroup for this table " + desc.getTableName() - + " does not exist."); - } - if (!rsGroupInfo.containsTable(desc.getTableName())) { - LOG.debug("Pre-moving table " + desc.getTableName() + " to RSGroup " + rsGroupInfo.getName()); - groupAdminServer.moveTables(Sets.newHashSet(desc.getTableName()), rsGroupInfo.getName()); - } - } - - ///////////////////////////////////////////////////////////////////////////// - // MasterObserver overrides - ///////////////////////////////////////////////////////////////////////////// - - @Override - public void preCreateTableAction( - final ObserverContext ctx, - final TableDescriptor desc, - final RegionInfo[] regions) throws IOException { - if (desc.getTableName().isSystemTable()) { - return; - } - RSGroupInfo rsGroupInfo = groupInfoManager.determineRSGroupInfoForTable(desc.getTableName()); - if (rsGroupInfo == null) { - throw new ConstraintException("Default RSGroup for this table " + desc.getTableName() - + " does not exist."); - } - if (!RSGroupUtil.rsGroupHasOnlineServer(master, rsGroupInfo)) { - throw new HBaseIOException("No online servers in the rsgroup " + rsGroupInfo.getName() - + " which table " + desc.getTableName().getNameAsString() + " belongs to"); - } - synchronized (groupInfoManager) { - groupInfoManager.moveTables( - Collections.singleton(desc.getTableName()), rsGroupInfo.getName()); - } - } - - // Remove table from its RSGroup. - @Override - public void postDeleteTable(ObserverContext ctx, - TableName tableName) throws IOException { - try { - RSGroupInfo group = groupAdminServer.getRSGroupInfoOfTable(tableName); - if (group != null) { - LOG.debug(String.format("Removing deleted table '%s' from rsgroup '%s'", tableName, - group.getName())); - groupAdminServer.moveTables(Sets.newHashSet(tableName), null); - } - } catch (IOException ex) { - LOG.debug("Failed to perform RSGroup information cleanup for table: " + tableName, ex); - } - } - - @Override - public void preCreateNamespace(ObserverContext ctx, - NamespaceDescriptor ns) throws IOException { - String group = ns.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); - if(group != null && groupAdminServer.getRSGroupInfo(group) == null) { - throw new ConstraintException("Region server group " + group + " does not exist."); - } - } - - @Override - public void preModifyNamespace(ObserverContext ctx, - NamespaceDescriptor ns) throws IOException { - preCreateNamespace(ctx, ns); - } - - @Override - public void preCloneSnapshot(ObserverContext ctx, - SnapshotDescription snapshot, TableDescriptor desc) throws IOException { - assignTableToGroup(desc); - } - - @Override - public void postClearDeadServers(ObserverContext ctx, - List servers, List notClearedServers) - throws IOException { - Set
clearedServer = servers.stream(). - filter(server -> !notClearedServers.contains(server)). - map(ServerName::getAddress). - collect(Collectors.toSet()); - if(!clearedServer.isEmpty()) { - groupAdminServer.removeServers(clearedServer); - } - } - - public void checkPermission(String request) throws IOException { - accessChecker.requirePermission(getActiveUser(), request, null, Action.ADMIN); - } - - /** - * Returns the active user to which authorization checks should be applied. - * If we are in the context of an RPC call, the remote user is used, - * otherwise the currently logged in user is used. - */ - private User getActiveUser() throws IOException { - // for non-rpc handling, fallback to system user - Optional optionalUser = RpcServer.getRequestUser(); - if (optionalUser.isPresent()) { - return optionalUser.get(); - } - return userProvider.getCurrent(); - } -} diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupProtobufUtil.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupProtobufUtil.java deleted file mode 100644 index 9b6ba1cc1c9e..000000000000 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupProtobufUtil.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.rsgroup; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.net.Address; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; -import org.apache.hadoop.hbase.protobuf.generated.TableProtos; -import org.apache.yetus.audience.InterfaceAudience; - -@InterfaceAudience.Private -final class RSGroupProtobufUtil { - private RSGroupProtobufUtil() { - } - - static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) { - RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName()); - for(HBaseProtos.ServerName el: proto.getServersList()) { - RSGroupInfo.addServer(Address.fromParts(el.getHostName(), el.getPort())); - } - for(TableProtos.TableName pTableName: proto.getTablesList()) { - RSGroupInfo.addTable(ProtobufUtil.toTableName(pTableName)); - } - return RSGroupInfo; - } - - static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) { - List tables = new ArrayList<>(pojo.getTables().size()); - for(TableName arg: pojo.getTables()) { - tables.add(ProtobufUtil.toProtoTableName(arg)); - } - List hostports = new ArrayList<>(pojo.getServers().size()); - for(Address el: pojo.getServers()) { - hostports.add(HBaseProtos.ServerName.newBuilder() - .setHostName(el.getHostname()) - .setPort(el.getPort()) - .build()); - } - return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName()) - .addAllServers(hostports) - .addAllTables(tables).build(); - } -} diff --git a/hbase-rsgroup/src/test/resources/log4j.properties b/hbase-rsgroup/src/test/resources/log4j.properties deleted file mode 100644 index c322699ced24..000000000000 --- a/hbase-rsgroup/src/test/resources/log4j.properties +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -hbase.root.logger=INFO,console -hbase.log.dir=. -hbase.log.file=hbase.log - -# Define the root logger to the system property "hbase.root.logger". -log4j.rootLogger=${hbase.root.logger} - -# Logging Threshold -log4j.threshold=ALL - -# -# Daily Rolling File Appender -# -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout -# Debugging Pattern format -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n - - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n - -# Custom Logging levels - -#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG - -log4j.logger.org.apache.hadoop=WARN -log4j.logger.org.apache.zookeeper=ERROR -log4j.logger.org.apache.hadoop.hbase=DEBUG - -#These settings are workarounds against spurious logs from the minicluster. -#See HBASE-4709 -log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN -log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN -log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN -log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN -# Enable this to get detailed connection error/retry logging. -# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java similarity index 100% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java similarity index 97% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java index 5e5d5907f744..8bf85354e365 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java @@ -18,12 +18,10 @@ package org.apache.hadoop.hbase.rsgroup; import com.google.protobuf.ServiceException; - import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Set; - import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.Admin; @@ -70,7 +68,7 @@ public RSGroupInfo getRSGroupInfo(String groupName) throws IOException { GetRSGroupInfoResponse resp = stub.getRSGroupInfo(null, GetRSGroupInfoRequest.newBuilder().setRSGroupName(groupName).build()); if(resp.hasRSGroupInfo()) { - return RSGroupProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); + return ProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); } return null; } catch (ServiceException e) { @@ -85,7 +83,7 @@ public RSGroupInfo getRSGroupInfoOfTable(TableName tableName) throws IOException try { GetRSGroupInfoOfTableResponse resp = stub.getRSGroupInfoOfTable(null, request); if (resp.hasRSGroupInfo()) { - return RSGroupProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); + return ProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); } return null; } catch (ServiceException e) { @@ -167,7 +165,7 @@ public List listRSGroups() throws IOException { ListRSGroupInfosRequest.getDefaultInstance()).getRSGroupInfoList(); List result = new ArrayList<>(resp.size()); for(RSGroupProtos.RSGroupInfo entry : resp) { - result.add(RSGroupProtobufUtil.toGroupInfo(entry)); + result.add(ProtobufUtil.toGroupInfo(entry)); } return result; } catch (ServiceException e) { @@ -186,7 +184,7 @@ public RSGroupInfo getRSGroupOfServer(Address hostPort) throws IOException { try { GetRSGroupInfoOfServerResponse resp = stub.getRSGroupInfoOfServer(null, request); if (resp.hasRSGroupInfo()) { - return RSGroupProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); + return ProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); } return null; } catch (ServiceException e) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java new file mode 100644 index 000000000000..66dca266afb1 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import com.google.protobuf.Service; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.PleaseHoldException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.constraint.ConstraintException; +import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor; +import org.apache.hadoop.hbase.coprocessor.HasMasterServices; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.MasterObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.security.access.AccessChecker; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.xiaomi.infra.thirdparty.com.google.common.annotations.VisibleForTesting; +import com.xiaomi.infra.thirdparty.com.google.common.collect.Sets; + +// TODO: Encapsulate MasterObserver functions into separate subclass. +@CoreCoprocessor +@InterfaceAudience.Private +public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver { + static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminEndpoint.class); + + private MasterServices master; + // Only instance of RSGroupInfoManager. RSGroup aware load balancers ask for this instance on + // their setup. + private RSGroupInfoManager groupInfoManager; + private RSGroupAdminServer groupAdminServer; + private RSGroupAdminServiceImpl groupAdminService = new RSGroupAdminServiceImpl(); + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + if (!(env instanceof HasMasterServices)) { + throw new IOException("Does not implement HMasterServices"); + } + + master = ((HasMasterServices) env).getMasterServices(); + groupInfoManager = RSGroupInfoManagerImpl.getInstance(master); + groupAdminServer = new RSGroupAdminServer(master, groupInfoManager); + Class clazz = + master.getConfiguration().getClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, null); + if (!RSGroupableBalancer.class.isAssignableFrom(clazz)) { + throw new IOException("Configured balancer does not support RegionServer groups."); + } + AccessChecker accessChecker = ((HasMasterServices) env).getMasterServices().getAccessChecker(); + + // set the user-provider. + UserProvider userProvider = UserProvider.instantiate(env.getConfiguration()); + groupAdminService.initialize(master, groupAdminServer, accessChecker, userProvider); + } + + @Override + public void stop(CoprocessorEnvironment env) { + } + + @Override + public Iterable getServices() { + return Collections.singleton(groupAdminService); + } + + @Override + public Optional getMasterObserver() { + return Optional.of(this); + } + + RSGroupInfoManager getGroupInfoManager() { + return groupInfoManager; + } + + @VisibleForTesting + RSGroupAdminServiceImpl getGroupAdminService() { + return groupAdminService; + } + + private void assignTableToGroup(TableDescriptor desc) throws IOException { + String groupName = + master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) + .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); + if (groupName == null) { + groupName = RSGroupInfo.DEFAULT_GROUP; + } + RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); + if (rsGroupInfo == null) { + throw new ConstraintException( + "Default RSGroup (" + groupName + ") for this table's namespace does not exist."); + } + if (!rsGroupInfo.containsTable(desc.getTableName())) { + LOG.debug("Pre-moving table " + desc.getTableName() + " to RSGroup " + groupName); + groupAdminServer.moveTables(Sets.newHashSet(desc.getTableName()), groupName); + } + } + + ///////////////////////////////////////////////////////////////////////////// + // MasterObserver overrides + ///////////////////////////////////////////////////////////////////////////// + + private boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException { + String groupName; + try { + groupName = master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) + .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); + if (groupName == null) { + groupName = RSGroupInfo.DEFAULT_GROUP; + } + } catch (MasterNotRunningException | PleaseHoldException e) { + LOG.info("Master has not initialized yet; temporarily using default RSGroup '" + + RSGroupInfo.DEFAULT_GROUP + "' for deploy of system table"); + groupName = RSGroupInfo.DEFAULT_GROUP; + } + + RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); + if (rsGroupInfo == null) { + throw new ConstraintException( + "Default RSGroup (" + groupName + ") for this table's " + "namespace does not exist."); + } + + for (ServerName onlineServer : master.getServerManager().createDestinationServersList()) { + if (rsGroupInfo.getServers().contains(onlineServer.getAddress())) { + return true; + } + } + return false; + } + + @Override + public void preCreateTableAction(final ObserverContext ctx, + final TableDescriptor desc, final RegionInfo[] regions) throws IOException { + if (!desc.getTableName().isSystemTable() && !rsgroupHasServersOnline(desc)) { + throw new HBaseIOException("No online servers in the rsgroup, which table " + + desc.getTableName().getNameAsString() + " belongs to"); + } + } + + // Assign table to default RSGroup. + @Override + public void postCreateTable(ObserverContext ctx, + TableDescriptor desc, RegionInfo[] regions) throws IOException { + assignTableToGroup(desc); + } + + // Remove table from its RSGroup. + @Override + public void postDeleteTable(ObserverContext ctx, + TableName tableName) throws IOException { + try { + RSGroupInfo group = groupAdminServer.getRSGroupInfoOfTable(tableName); + if (group != null) { + LOG.debug(String.format("Removing deleted table '%s' from rsgroup '%s'", tableName, + group.getName())); + groupAdminServer.moveTables(Sets.newHashSet(tableName), null); + } + } catch (IOException ex) { + LOG.debug("Failed to perform RSGroup information cleanup for table: " + tableName, ex); + } + } + + @Override + public void preCreateNamespace(ObserverContext ctx, + NamespaceDescriptor ns) throws IOException { + String group = ns.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); + if (group != null && groupAdminServer.getRSGroupInfo(group) == null) { + throw new ConstraintException("Region server group " + group + " does not exit"); + } + } + + @Override + public void preModifyNamespace(ObserverContext ctx, + NamespaceDescriptor currentNsDesc, NamespaceDescriptor newNsDesc) throws IOException { + preCreateNamespace(ctx, newNsDesc); + } + + @Override + public void preCloneSnapshot(ObserverContext ctx, + SnapshotDescription snapshot, TableDescriptor desc) throws IOException { + assignTableToGroup(desc); + } + + @Override + public void postClearDeadServers(ObserverContext ctx, + List servers, List notClearedServers) throws IOException { + Set
clearedServer = + servers.stream().filter(server -> !notClearedServers.contains(server)) + .map(ServerName::getAddress).collect(Collectors.toSet()); + if (!clearedServer.isEmpty()) { + groupAdminServer.removeServers(clearedServer); + } + } +} diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java similarity index 100% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServiceImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServiceImpl.java new file mode 100644 index 000000000000..df3459285fd7 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServiceImpl.java @@ -0,0 +1,356 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import com.google.protobuf.RpcCallback; +import com.google.protobuf.RpcController; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; +import org.apache.hadoop.hbase.ipc.RpcServer; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersResponse; +import org.apache.hadoop.hbase.protobuf.generated.TableProtos; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.security.access.AccessChecker; +import org.apache.hadoop.hbase.security.access.Permission.Action; + +import com.xiaomi.infra.thirdparty.com.google.common.annotations.VisibleForTesting; +import com.xiaomi.infra.thirdparty.com.google.common.collect.Sets; + +/** + * Implementation of RSGroupAdminService defined in RSGroupAdmin.proto. This class calls + * {@link RSGroupAdminServer} for actual work, converts result to protocol buffer response, handles + * exceptions if any occurred and then calls the {@code RpcCallback} with the response. + */ +class RSGroupAdminServiceImpl extends RSGroupAdminProtos.RSGroupAdminService { + + private MasterServices master; + + private RSGroupAdminServer groupAdminServer; + + private AccessChecker accessChecker; + + /** Provider for mapping principal names to Users */ + private UserProvider userProvider; + + RSGroupAdminServiceImpl() { + } + + void initialize(MasterServices master, RSGroupAdminServer groupAdminServer, + AccessChecker accessChecker, UserProvider userProvider) { + this.master = master; + this.groupAdminServer = groupAdminServer; + this.accessChecker = accessChecker; + this.userProvider = userProvider; + } + + @VisibleForTesting + void checkPermission(String request) throws IOException { + accessChecker.requirePermission(getActiveUser(), request, null, Action.ADMIN); + } + + /** + * Returns the active user to which authorization checks should be applied. If we are in the + * context of an RPC call, the remote user is used, otherwise the currently logged in user is + * used. + */ + private User getActiveUser() throws IOException { + // for non-rpc handling, fallback to system user + Optional optionalUser = RpcServer.getRequestUser(); + if (optionalUser.isPresent()) { + return optionalUser.get(); + } + return userProvider.getCurrent(); + } + + @Override + public void getRSGroupInfo(RpcController controller, GetRSGroupInfoRequest request, + RpcCallback done) { + GetRSGroupInfoResponse.Builder builder = GetRSGroupInfoResponse.newBuilder(); + String groupName = request.getRSGroupName(); + RSGroupAdminEndpoint.LOG.info( + master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, group=" + groupName); + try { + checkPermission("getRSGroupInfo"); + RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); + if (rsGroupInfo != null) { + builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(rsGroupInfo)); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void getRSGroupInfoOfTable(RpcController controller, GetRSGroupInfoOfTableRequest request, + RpcCallback done) { + GetRSGroupInfoOfTableResponse.Builder builder = GetRSGroupInfoOfTableResponse.newBuilder(); + TableName tableName = ProtobufUtil.toTableName(request.getTableName()); + RSGroupAdminEndpoint.LOG.info( + master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, table=" + tableName); + try { + checkPermission("getRSGroupInfoOfTable"); + RSGroupInfo RSGroupInfo = groupAdminServer.getRSGroupInfoOfTable(tableName); + if (RSGroupInfo != null) { + builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(RSGroupInfo)); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void moveServers(RpcController controller, MoveServersRequest request, + RpcCallback done) { + MoveServersResponse.Builder builder = MoveServersResponse.newBuilder(); + Set
hostPorts = Sets.newHashSet(); + for (HBaseProtos.ServerName el : request.getServersList()) { + hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); + } + RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts + + " to rsgroup " + request.getTargetGroup()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preMoveServers(hostPorts, request.getTargetGroup()); + } + checkPermission("moveServers"); + groupAdminServer.moveServers(hostPorts, request.getTargetGroup()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postMoveServers(hostPorts, request.getTargetGroup()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void moveTables(RpcController controller, MoveTablesRequest request, + RpcCallback done) { + MoveTablesResponse.Builder builder = MoveTablesResponse.newBuilder(); + Set tables = new HashSet<>(request.getTableNameList().size()); + for (TableProtos.TableName tableName : request.getTableNameList()) { + tables.add(ProtobufUtil.toTableName(tableName)); + } + RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " move tables " + tables + + " to rsgroup " + request.getTargetGroup()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preMoveTables(tables, request.getTargetGroup()); + } + checkPermission("moveTables"); + groupAdminServer.moveTables(tables, request.getTargetGroup()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postMoveTables(tables, request.getTargetGroup()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void addRSGroup(RpcController controller, AddRSGroupRequest request, + RpcCallback done) { + AddRSGroupResponse.Builder builder = AddRSGroupResponse.newBuilder(); + RSGroupAdminEndpoint.LOG + .info(master.getClientIdAuditPrefix() + " add rsgroup " + request.getRSGroupName()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preAddRSGroup(request.getRSGroupName()); + } + checkPermission("addRSGroup"); + groupAdminServer.addRSGroup(request.getRSGroupName()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postAddRSGroup(request.getRSGroupName()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void removeRSGroup(RpcController controller, RemoveRSGroupRequest request, + RpcCallback done) { + RemoveRSGroupResponse.Builder builder = RemoveRSGroupResponse.newBuilder(); + RSGroupAdminEndpoint.LOG + .info(master.getClientIdAuditPrefix() + " remove rsgroup " + request.getRSGroupName()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preRemoveRSGroup(request.getRSGroupName()); + } + checkPermission("removeRSGroup"); + groupAdminServer.removeRSGroup(request.getRSGroupName()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postRemoveRSGroup(request.getRSGroupName()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void balanceRSGroup(RpcController controller, BalanceRSGroupRequest request, + RpcCallback done) { + BalanceRSGroupResponse.Builder builder = BalanceRSGroupResponse.newBuilder(); + RSGroupAdminEndpoint.LOG.info( + master.getClientIdAuditPrefix() + " balance rsgroup, group=" + request.getRSGroupName()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preBalanceRSGroup(request.getRSGroupName()); + } + checkPermission("balanceRSGroup"); + boolean balancerRan = groupAdminServer.balanceRSGroup(request.getRSGroupName()); + builder.setBalanceRan(balancerRan); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postBalanceRSGroup(request.getRSGroupName(), balancerRan); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + builder.setBalanceRan(false); + } + done.run(builder.build()); + } + + @Override + public void listRSGroupInfos(RpcController controller, ListRSGroupInfosRequest request, + RpcCallback done) { + ListRSGroupInfosResponse.Builder builder = ListRSGroupInfosResponse.newBuilder(); + RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " list rsgroup"); + try { + checkPermission("listRSGroup"); + for (RSGroupInfo RSGroupInfo : groupAdminServer.listRSGroups()) { + builder.addRSGroupInfo(ProtobufUtil.toProtoGroupInfo(RSGroupInfo)); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void getRSGroupInfoOfServer(RpcController controller, + GetRSGroupInfoOfServerRequest request, RpcCallback done) { + GetRSGroupInfoOfServerResponse.Builder builder = GetRSGroupInfoOfServerResponse.newBuilder(); + Address hp = + Address.fromParts(request.getServer().getHostName(), request.getServer().getPort()); + RSGroupAdminEndpoint.LOG + .info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, server=" + hp); + try { + checkPermission("getRSGroupInfoOfServer"); + RSGroupInfo info = groupAdminServer.getRSGroupOfServer(hp); + if (info != null) { + builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(info)); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void moveServersAndTables(RpcController controller, MoveServersAndTablesRequest request, + RpcCallback done) { + MoveServersAndTablesResponse.Builder builder = MoveServersAndTablesResponse.newBuilder(); + Set
hostPorts = Sets.newHashSet(); + for (HBaseProtos.ServerName el : request.getServersList()) { + hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); + } + Set tables = new HashSet<>(request.getTableNameList().size()); + for (TableProtos.TableName tableName : request.getTableNameList()) { + tables.add(ProtobufUtil.toTableName(tableName)); + } + RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts + + " and tables " + tables + " to rsgroup" + request.getTargetGroup()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preMoveServersAndTables(hostPorts, tables, + request.getTargetGroup()); + } + checkPermission("moveServersAndTables"); + groupAdminServer.moveServersAndTables(hostPorts, tables, request.getTargetGroup()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postMoveServersAndTables(hostPorts, tables, + request.getTargetGroup()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void removeServers(RpcController controller, RemoveServersRequest request, + RpcCallback done) { + RemoveServersResponse.Builder builder = RemoveServersResponse.newBuilder(); + Set
servers = Sets.newHashSet(); + for (HBaseProtos.ServerName el : request.getServersList()) { + servers.add(Address.fromParts(el.getHostName(), el.getPort())); + } + RSGroupAdminEndpoint.LOG.info( + master.getClientIdAuditPrefix() + " remove decommissioned servers from rsgroup: " + servers); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preRemoveServers(servers); + } + checkPermission("removeServers"); + groupAdminServer.removeServers(servers); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postRemoveServers(servers); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } +} \ No newline at end of file diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java similarity index 97% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index d9504d2eaa34..ea6a728c59d9 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.rsgroup; import java.io.IOException; @@ -53,18 +52,17 @@ import com.xiaomi.infra.thirdparty.com.google.common.collect.Maps; /** - * GroupBasedLoadBalancer, used when Region Server Grouping is configured (HBase-6721) - * It does region balance based on a table's group membership. - * - * Most assignment methods contain two exclusive code paths: Online - when the group - * table is online and Offline - when it is unavailable. - * - * During Offline, assignments are assigned based on cached information in zookeeper. - * If unavailable (ie bootstrap) then regions are assigned randomly. - * - * Once the GROUP table has been assigned, the balancer switches to Online and will then - * start providing appropriate assignments for user tables. - * + * GroupBasedLoadBalancer, used when Region Server Grouping is configured (HBase-6721) It does + * region balance based on a table's group membership. + *

+ * Most assignment methods contain two exclusive code paths: Online - when the group table is online + * and Offline - when it is unavailable. + *

+ * During Offline, assignments are assigned based on cached information in zookeeper. If unavailable + * (ie bootstrap) then regions are assigned randomly. + *

+ * Once the GROUP table has been assigned, the balancer switches to Online and will then start + * providing appropriate assignments for user tables. */ @InterfaceAudience.Private public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { @@ -116,8 +114,8 @@ public void setMasterServices(MasterServices masterServices) { public List balanceCluster( Map>> loadOfAllTable) throws IOException { if (!isOnline()) { - throw new ConstraintException(RSGroupInfoManager.RSGROUP_TABLE_NAME + - " is not online, unable to perform balance"); + throw new ConstraintException( + RSGroupInfoManager.class.getSimpleName() + " is not online, unable to perform balance"); } // Calculate correct assignments and a list of RegionPlan for mis-placed regions Pair>>, List> diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java similarity index 84% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java index 594f0f005da7..f3de4bea488b 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java @@ -15,39 +15,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.rsgroup; import java.io.IOException; import java.util.List; import java.util.Set; - -import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.net.Address; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; /** - * Interface used to manage RSGroupInfo storage. An implementation - * has the option to support offline mode. - * See {@link RSGroupBasedLoadBalancer} + * Interface used to manage RSGroupInfo storage. An implementation has the option to support offline + * mode. See {@code RSGroupBasedLoadBalancer}. */ @InterfaceAudience.Private public interface RSGroupInfoManager { - String REASSIGN_WAIT_INTERVAL_KEY = "hbase.rsgroup.reassign.wait"; - long DEFAULT_REASSIGN_WAIT_INTERVAL = 30 * 1000L; - - //Assigned before user tables - TableName RSGROUP_TABLE_NAME = - TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "rsgroup"); - String rsGroupZNode = "rsgroup"; - byte[] META_FAMILY_BYTES = Bytes.toBytes("m"); - byte[] META_QUALIFIER_BYTES = Bytes.toBytes("i"); - byte[] ROW_KEY = {0}; - void start(); /** @@ -87,7 +71,6 @@ Set

moveServers(Set
servers, String srcGroup, String dstGroup) /** * Set the group membership of a set of tables - * * @param tableNames set of tables to move * @param groupName name of group of tables to move to */ @@ -105,7 +88,6 @@ Set
moveServers(Set
servers, String srcGroup, String dstGroup) /** * Whether the manager is able to fully return group metadata - * * @return whether the manager is in online mode */ boolean isOnline(); @@ -117,8 +99,8 @@ Set
moveServers(Set
servers, String srcGroup, String dstGroup) * @param srcGroup groupName being moved from * @param dstGroup groupName being moved to */ - void moveServersAndTables(Set
servers, Set tables, - String srcGroup, String dstGroup) throws IOException; + void moveServersAndTables(Set
servers, Set tables, String srcGroup, + String dstGroup) throws IOException; /** * Remove decommissioned servers from rsgroup diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java similarity index 96% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java index d85df00ac8b6..1a69e83bb092 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java @@ -92,13 +92,13 @@ * RSGroupInfo Map at {@link #rsGroupMap} and a Map of tables to the name of the rsgroup they belong * too (in {@link #tableMap}). These Maps are persisted to the hbase:rsgroup table (and cached in * zk) on each modification. - *

+ *

* Mutations on state are synchronized but reads can continue without having to wait on an instance * monitor, mutations do wholesale replace of the Maps on update -- Copy-On-Write; the local Maps of * state are read-only, just-in-case (see flushConfig). - *

+ *

* Reads must not block else there is a danger we'll deadlock. - *

+ *

* Clients of this class, the {@link RSGroupAdminEndpoint} for example, want to query and then act * on the results of the query modifying cache in zookeeper without another thread making * intermediate modifications. These clients synchronize on the 'this' instance so no other has @@ -108,6 +108,24 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { private static final Logger LOG = LoggerFactory.getLogger(RSGroupInfoManagerImpl.class); + private static final String REASSIGN_WAIT_INTERVAL_KEY = "hbase.rsgroup.reassign.wait"; + private static final long DEFAULT_REASSIGN_WAIT_INTERVAL = 30 * 1000L; + + // Assigned before user tables + @VisibleForTesting + static final TableName RSGROUP_TABLE_NAME = + TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "rsgroup"); + + private static final String RS_GROUP_ZNODE = "rsgroup"; + + @VisibleForTesting + static final byte[] META_FAMILY_BYTES = Bytes.toBytes("m"); + + @VisibleForTesting + static final byte[] META_QUALIFIER_BYTES = Bytes.toBytes("i"); + + private static final byte[] ROW_KEY = { 0 }; + /** Table descriptor for hbase:rsgroup catalog table */ private static final TableDescriptor RSGROUP_TABLE_DESC; static { @@ -188,6 +206,7 @@ private RSGroupInfoManagerImpl(MasterServices masterServices) throws IOException script = new RSGroupMappingScript(masterServices.getConfiguration()); } + private synchronized void init() throws IOException { refresh(); serverEventsListenerThread.start(); @@ -452,7 +471,7 @@ public RSGroupInfo determineRSGroupInfoForTable(TableName tableName) return getRSGroup(RSGroupInfo.DEFAULT_GROUP); } - List retrieveGroupListFromGroupTable() throws IOException { + private List retrieveGroupListFromGroupTable() throws IOException { List rsGroupInfoList = Lists.newArrayList(); try (Table table = conn.getTable(RSGROUP_TABLE_NAME); ResultScanner scanner = table.getScanner(new Scan())) { @@ -463,14 +482,14 @@ List retrieveGroupListFromGroupTable() throws IOException { } RSGroupProtos.RSGroupInfo proto = RSGroupProtos.RSGroupInfo .parseFrom(result.getValue(META_FAMILY_BYTES, META_QUALIFIER_BYTES)); - rsGroupInfoList.add(RSGroupProtobufUtil.toGroupInfo(proto)); + rsGroupInfoList.add(ProtobufUtil.toGroupInfo(proto)); } } return rsGroupInfoList; } - List retrieveGroupListFromZookeeper() throws IOException { - String groupBasePath = ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, rsGroupZNode); + private List retrieveGroupListFromZookeeper() throws IOException { + String groupBasePath = ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, RS_GROUP_ZNODE); List RSGroupInfoList = Lists.newArrayList(); // Overwrite any info stored by table, this takes precedence try { @@ -486,7 +505,7 @@ List retrieveGroupListFromZookeeper() throws IOException { ByteArrayInputStream bis = new ByteArrayInputStream(data, ProtobufUtil.lengthOfPBMagic(), data.length); RSGroupInfoList - .add(RSGroupProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis))); + .add(ProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis))); } } LOG.debug("Read ZK GroupInfo count:" + RSGroupInfoList.size()); @@ -562,7 +581,7 @@ private synchronized Map flushConfigTable(Map newGroupMap) thro resetRSGroupAndTableMaps(newGroupMap, newTableMap); try { - String groupBasePath = ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, rsGroupZNode); + String groupBasePath = + ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, RS_GROUP_ZNODE); ZKUtil.createAndFailSilent(watcher, groupBasePath, ProtobufMagic.PB_MAGIC); List zkOps = new ArrayList<>(newGroupMap.size()); @@ -635,7 +655,7 @@ private synchronized void flushConfig(Map newGroupMap) thro for (RSGroupInfo RSGroupInfo : newGroupMap.values()) { String znode = ZNodePaths.joinZNode(groupBasePath, RSGroupInfo.getName()); - RSGroupProtos.RSGroupInfo proto = RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo); + RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo); LOG.debug("Updating znode: " + znode); ZKUtil.createAndFailSilent(watcher, znode); zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode)); @@ -851,8 +871,8 @@ private void createRSGroupTable() throws IOException { } else { Procedure result = masterServices.getMasterProcedureExecutor().getResult(procId); if (result != null && result.isFailed()) { - throw new IOException( - "Failed to create group table. " + MasterProcedureUtil.unwrapRemoteIOException(result)); + throw new IOException("Failed to create group table. " + + MasterProcedureUtil.unwrapRemoteIOException(result)); } } } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java similarity index 100% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupUtil.java similarity index 100% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupUtil.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupUtil.java diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java similarity index 100% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSCPWithReplicasWithRSGroup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSCPWithReplicasWithRSGroup.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSCPWithReplicasWithRSGroup.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSCPWithReplicasWithRSGroup.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestDetermineRSGroupInfoForTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestDetermineRSGroupInfoForTable.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestDetermineRSGroupInfoForTable.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestDetermineRSGroupInfoForTable.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMappingScript.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMappingScript.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMappingScript.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMappingScript.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupUtil.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupUtil.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupUtil.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java similarity index 98% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java index 567f3c6da399..5e3e70708b2b 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java @@ -140,7 +140,7 @@ public boolean evaluate() throws Exception { } }); // Move table to group and wait. - groupAdmin.moveTables(Sets.newHashSet(RSGroupInfoManager.RSGROUP_TABLE_NAME), newGroup); + groupAdmin.moveTables(Sets.newHashSet(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME), newGroup); LOG.info("Waiting for move table..."); TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override @@ -169,7 +169,7 @@ public boolean evaluate() throws Exception { // Make sure balancer is in offline mode, since this is what we're testing. assertFalse(groupMgr.isOnline()); // Verify the group affiliation that's loaded from ZK instead of tables. - assertEquals(newGroup, groupMgr.getRSGroupOfTable(RSGroupInfoManager.RSGROUP_TABLE_NAME)); + assertEquals(newGroup, groupMgr.getRSGroupOfTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME)); assertEquals(RSGroupInfo.DEFAULT_GROUP, groupMgr.getRSGroupOfTable(failoverTable)); // Kill final regionserver to see the failover happens for all tables except GROUP table since // it's group does not have any online RS. @@ -182,7 +182,7 @@ public boolean evaluate() throws Exception { return failoverRS.getRegions(failoverTable).size() >= 1; } }); - Assert.assertEquals(0, failoverRS.getRegions(RSGroupInfoManager.RSGROUP_TABLE_NAME).size()); + Assert.assertEquals(0, failoverRS.getRegions(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME).size()); // Need this for minicluster to shutdown cleanly. master.stopMaster(); diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java similarity index 71% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java index 77e4e493e8ea..2c3e14d58381 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java @@ -51,11 +51,11 @@ import org.slf4j.LoggerFactory; /** - * Performs authorization checks for rsgroup operations, according to different - * levels of authorized users. + * Performs authorization checks for rsgroup operations, according to different levels of authorized + * users. */ -@Category({SecurityTests.class, MediumTests.class}) -public class TestRSGroupsWithACL extends SecureTestUtil{ +@Category({ SecurityTests.class, MediumTests.class }) +public class TestRSGroupsWithACL extends SecureTestUtil { @ClassRule public static final HBaseClassTestRule CLASS_RULE = @@ -100,8 +100,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil{ public static void setupBeforeClass() throws Exception { // setup configuration conf = TEST_UTIL.getConfiguration(); - conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, - RSGroupBasedLoadBalancer.class.getName()); + conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); // Enable security enableSecurity(conf); // Verify enableSecurity sets up what we require @@ -110,6 +109,8 @@ public static void setupBeforeClass() throws Exception { configureRSGroupAdminEndpoint(conf); TEST_UTIL.startMiniCluster(); + rsGroupAdminEndpoint = (RSGroupAdminEndpoint) TEST_UTIL.getMiniHBaseCluster().getMaster() + .getMasterCoprocessorHost().findCoprocessor(RSGroupAdminEndpoint.class.getName()); HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); TEST_UTIL.waitFor(60000, (Predicate) () -> @@ -148,31 +149,21 @@ private static void setUpTableAndUserPermissions() throws Exception { cfd.setMaxVersions(100); tableBuilder.setColumnFamily(cfd.build()); tableBuilder.setValue(TableDescriptorBuilder.OWNER, USER_OWNER.getShortName()); - createTable(TEST_UTIL, tableBuilder.build(), - new byte[][] { Bytes.toBytes("s") }); + createTable(TEST_UTIL, tableBuilder.build(), new byte[][] { Bytes.toBytes("s") }); // Set up initial grants - grantGlobal(TEST_UTIL, USER_ADMIN.getShortName(), - Permission.Action.ADMIN, - Permission.Action.CREATE, - Permission.Action.READ, - Permission.Action.WRITE); + grantGlobal(TEST_UTIL, USER_ADMIN.getShortName(), Permission.Action.ADMIN, + Permission.Action.CREATE, Permission.Action.READ, Permission.Action.WRITE); - grantOnTable(TEST_UTIL, USER_RW.getShortName(), - TEST_TABLE, TEST_FAMILY, null, - Permission.Action.READ, - Permission.Action.WRITE); + grantOnTable(TEST_UTIL, USER_RW.getShortName(), TEST_TABLE, TEST_FAMILY, null, + Permission.Action.READ, Permission.Action.WRITE); // USER_CREATE is USER_RW plus CREATE permissions - grantOnTable(TEST_UTIL, USER_CREATE.getShortName(), - TEST_TABLE, null, null, - Permission.Action.CREATE, - Permission.Action.READ, - Permission.Action.WRITE); + grantOnTable(TEST_UTIL, USER_CREATE.getShortName(), TEST_TABLE, null, null, + Permission.Action.CREATE, Permission.Action.READ, Permission.Action.WRITE); - grantOnTable(TEST_UTIL, USER_RO.getShortName(), - TEST_TABLE, TEST_FAMILY, null, - Permission.Action.READ); + grantOnTable(TEST_UTIL, USER_RO.getShortName(), TEST_TABLE, TEST_FAMILY, null, + Permission.Action.READ); grantGlobal(TEST_UTIL, toGroupEntry(GROUP_ADMIN), Permission.Action.ADMIN); grantGlobal(TEST_UTIL, toGroupEntry(GROUP_CREATE), Permission.Action.CREATE); @@ -181,8 +172,8 @@ private static void setUpTableAndUserPermissions() throws Exception { assertEquals(4, PermissionStorage.getTablePermissions(conf, TEST_TABLE).size()); try { - assertEquals(4, AccessControlClient.getUserPermissions(systemUserConnection, - TEST_TABLE.toString()).size()); + assertEquals(4, + AccessControlClient.getUserPermissions(systemUserConnection, TEST_TABLE.toString()).size()); } catch (AssertionError e) { fail(e.getMessage()); } catch (Throwable e) { @@ -217,125 +208,110 @@ private static void configureRSGroupAdminEndpoint(Configuration conf) { coprocessors += "," + currentCoprocessors; } conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, coprocessors); - conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, - RSGroupBasedLoadBalancer.class.getName()); + conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); } @Test public void testGetRSGroupInfo() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("getRSGroupInfo"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("getRSGroupInfo"); return null; }; - verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); - verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, - USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); + validateAdminPermissions(action); } @Test public void testGetRSGroupInfoOfTable() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("getRSGroupInfoOfTable"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("getRSGroupInfoOfTable"); return null; }; - verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); - verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, - USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); + validateAdminPermissions(action); } @Test public void testMoveServers() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("moveServers"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("moveServers"); return null; }; - verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); - verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, - USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); + validateAdminPermissions(action); } @Test public void testMoveTables() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("moveTables"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("moveTables"); return null; }; - verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); - verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, - USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); + validateAdminPermissions(action); } @Test public void testAddRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("addRSGroup"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("addRSGroup"); return null; }; - verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); - verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, - USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); + validateAdminPermissions(action); } @Test public void testRemoveRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("removeRSGroup"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("removeRSGroup"); return null; }; - verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); - verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, - USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); + validateAdminPermissions(action); } @Test public void testBalanceRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("balanceRSGroup"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("balanceRSGroup"); return null; }; - verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); - verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, - USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); + validateAdminPermissions(action); } @Test public void testListRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("listRSGroup"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("listRSGroup"); return null; }; - verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); - verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, - USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); + validateAdminPermissions(action); } @Test public void testGetRSGroupInfoOfServer() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("getRSGroupInfoOfServer"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("getRSGroupInfoOfServer"); return null; }; - verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); - verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, - USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); + validateAdminPermissions(action); } @Test public void testMoveServersAndTables() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("moveServersAndTables"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("moveServersAndTables"); return null; }; + validateAdminPermissions(action); + } + + private void validateAdminPermissions(AccessTestAction action) throws Exception { verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java similarity index 93% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java index 055f97d031f6..3786e9df4f6b 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java @@ -22,7 +22,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -53,7 +52,7 @@ public VerifyingRSGroupAdminClient(RSGroupAdmin RSGroupAdmin, Configuration conf throws IOException { wrapped = RSGroupAdmin; table = ConnectionFactory.createConnection(conf) - .getTable(RSGroupInfoManager.RSGROUP_TABLE_NAME); + .getTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME); zkw = new ZKWatcher(conf, this.getClass().getSimpleName(), null); } @@ -108,7 +107,7 @@ public RSGroupInfo getRSGroupOfServer(Address hostPort) throws IOException { @Override public void moveServersAndTables(Set

servers, Set tables, String targetGroup) - throws IOException { + throws IOException { wrapped.moveServersAndTables(servers, tables, targetGroup); verify(); } @@ -127,9 +126,9 @@ public void verify() throws IOException { RSGroupProtos.RSGroupInfo proto = RSGroupProtos.RSGroupInfo.parseFrom( result.getValue( - RSGroupInfoManager.META_FAMILY_BYTES, - RSGroupInfoManager.META_QUALIFIER_BYTES)); - groupMap.put(proto.getName(), RSGroupProtobufUtil.toGroupInfo(proto)); + RSGroupInfoManagerImpl.META_FAMILY_BYTES, + RSGroupInfoManagerImpl.META_QUALIFIER_BYTES)); + groupMap.put(proto.getName(), ProtobufUtil.toGroupInfo(proto)); } Assert.assertEquals(Sets.newHashSet(groupMap.values()), Sets.newHashSet(wrapped.listRSGroups())); @@ -141,7 +140,7 @@ public void verify() throws IOException { ProtobufUtil.expectPBMagicPrefix(data); ByteArrayInputStream bis = new ByteArrayInputStream( data, ProtobufUtil.lengthOfPBMagic(), data.length); - zList.add(RSGroupProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis))); + zList.add(ProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis))); } } Assert.assertEquals(zList.size(), groupMap.size()); diff --git a/hbase-shell/pom.xml b/hbase-shell/pom.xml index 29b098145d3b..6fdfd031c103 100644 --- a/hbase-shell/pom.xml +++ b/hbase-shell/pom.xml @@ -201,41 +201,6 @@ - - rsgroup - - - !skip-rsgroup - - - - - org.apache.hbase - hbase-rsgroup - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-test-source - - add-test-source - - - - src/test/rsgroup - - - - - - - - skipShellTests diff --git a/pom.xml b/pom.xml index 719e4abd8cdc..0b729152302c 100755 --- a/pom.xml +++ b/pom.xml @@ -1399,7 +1399,6 @@ hbase-procedure-${project.version}-tests.jar hbase-it-${project.version}-tests.jar hbase-annotations-${project.version}-tests.jar - hbase-rsgroup-${project.version}-tests.jar hbase-mapreduce-${project.version}-tests.jar hbase-zookeeper-${project.version}-tests.jar bash @@ -1529,18 +1528,6 @@ test-jar test - - hbase-rsgroup - org.apache.hbase - ${project.version} - - - hbase-rsgroup - org.apache.hbase - ${project.version} - test-jar - test - hbase-replication org.apache.hbase @@ -2200,17 +2187,6 @@ --> - - rsgroup - - - !skip-rsgroup - - - - hbase-rsgroup - - build-with-jdk8