From 0fae6670eb4bd316695dc1012830717f1b81e3a8 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Mon, 21 Dec 2020 20:58:15 +0800 Subject: [PATCH 1/3] fix hbase/rocksdb shard bug Change-Id: Ifce3d450712104c067a57c361351c9e047bc0cfa --- .../java/com/baidu/hugegraph/backend/store/BackendTable.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendTable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendTable.java index 5ba0a1dc97..cc86864cc9 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendTable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendTable.java @@ -254,8 +254,9 @@ public List splitEven(int count) { length); byte[] offset = start; byte[] last = offset; + boolean finished = false; List shards = new ArrayList<>(count); - while (Bytes.compare(offset, end) < 0) { + while (Bytes.compare(offset, end) < 0 && !finished) { offset = add(offset, each); if (offset.length > end.length || Bytes.compare(offset, end) > 0) { @@ -267,6 +268,7 @@ public List splitEven(int count) { } if (endChanged && Arrays.equals(offset, end)) { offset = this.endKey; + finished = true; } shards.add(new Shard(startKey(last), endKey(offset), 0)); last = offset; From 809feb3833d5e167aa5d5af9e2189b5a9c714954 Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Wed, 23 Dec 2020 21:25:14 +0800 Subject: [PATCH 2/3] add unit tests for shard range split Change-Id: I75b33c8cff9c7e48fc8f44e9e175f9365a143ee3 --- .../hugegraph/backend/store/BackendTable.java | 3 +- .../baidu/hugegraph/unit/UnitTestSuite.java | 2 + .../baidu/hugegraph/unit/core/RangeTest.java | 172 ++++++++++++++++++ 3 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/RangeTest.java diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendTable.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendTable.java index cc86864cc9..280e8187aa 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendTable.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendTable.java @@ -204,7 +204,7 @@ protected long maxKey() { protected abstract long estimateNumKeys(Session session); - protected static class Range { + public static class Range { private byte[] startKey; private byte[] endKey; @@ -247,7 +247,6 @@ public List splitEven(int count) { } assert count > 1; - assert startChanged != endChanged; byte[] each = align(new BigInteger(1, subtract(end, start)) .divide(BigInteger.valueOf(count)) .toByteArray(), diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java index c6ec329bb5..8f340ed999 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/UnitTestSuite.java @@ -38,6 +38,7 @@ import com.baidu.hugegraph.unit.core.ExceptionTest; import com.baidu.hugegraph.unit.core.LocksTableTest; import com.baidu.hugegraph.unit.core.QueryTest; +import com.baidu.hugegraph.unit.core.RangeTest; import com.baidu.hugegraph.unit.core.RolePermissionTest; import com.baidu.hugegraph.unit.core.RowLockTest; import com.baidu.hugegraph.unit.core.SecurityManagerTest; @@ -91,6 +92,7 @@ ConditionTest.class, ConditionQueryFlattenTest.class, QueryTest.class, + RangeTest.class, SecurityManagerTest.class, RolePermissionTest.class, ExceptionTest.class, diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/RangeTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/RangeTest.java new file mode 100644 index 0000000000..d7b16b5f3a --- /dev/null +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/core/RangeTest.java @@ -0,0 +1,172 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.unit.core; + +import java.util.List; + +import org.apache.logging.log4j.util.Strings; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.baidu.hugegraph.backend.store.BackendTable.ShardSpliter.Range; +import com.baidu.hugegraph.backend.store.Shard; +import com.baidu.hugegraph.testutil.Assert; + +public class RangeTest { + + private static final byte[] START = new byte[]{0x0}; + private static final byte[] END = new byte[]{ + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1 + }; + + @Before + public void setup() { + // pass + } + + @After + public void teardown() { + // pass + } + + @Test + public void testRangeOfOnlyOneRegion() { + // The startKey and endKey is "", if it's the only region of table + Range range = new Range(START, END); + + List shards = range.splitEven(0); + Assert.assertEquals(1, shards.size()); + Assert.assertEquals(Strings.EMPTY, shards.get(0).start()); + Assert.assertEquals(Strings.EMPTY, shards.get(0).end()); + + shards = range.splitEven(1); + Assert.assertEquals(1, shards.size()); + Assert.assertEquals(Strings.EMPTY, shards.get(0).start()); + Assert.assertEquals(Strings.EMPTY, shards.get(0).end()); + + shards = range.splitEven(3); + Assert.assertEquals(3, shards.size()); + Assert.assertEquals(Strings.EMPTY, shards.get(0).start()); + Assert.assertEquals("VVVVVVVVVVVVVVVVVVVVVQ==", shards.get(0).end()); + Assert.assertEquals("VVVVVVVVVVVVVVVVVVVVVQ==", shards.get(1).start()); + Assert.assertEquals("qqqqqqqqqqqqqqqqqqqqqg==", shards.get(1).end()); + Assert.assertEquals("qqqqqqqqqqqqqqqqqqqqqg==", shards.get(2).start()); + Assert.assertEquals(Strings.EMPTY, shards.get(2).end()); + + for (int i = 4; i < 100; i++) { + range.splitEven(i); + } + } + + @Test + public void testRangeOfRegionWithEndKey() { + byte[] end = new byte[]{-3, 0x31, 0x30, 0x30, 0x30, 0x77, + 0x20, 0x09, 0x38, 0x31, 0x33, 0x32, 0x35}; + Range range = new Range(START, end); + + List shards = range.splitEven(0); + Assert.assertEquals(1, shards.size()); + Assert.assertEquals(Strings.EMPTY, shards.get(0).start()); + Assert.assertEquals("/TEwMDB3IAk4MTMyNQ==", shards.get(0).end()); + + shards = range.splitEven(1); + Assert.assertEquals(1, shards.size()); + Assert.assertEquals(Strings.EMPTY, shards.get(0).start()); + Assert.assertEquals("/TEwMDB3IAk4MTMyNQ==", shards.get(0).end()); + + shards = range.splitEven(2); + Assert.assertEquals(3, shards.size()); + Assert.assertEquals(Strings.EMPTY, shards.get(0).start()); + Assert.assertEquals("fpiYGBg7kAScGJmZGg==", shards.get(0).end()); + Assert.assertEquals("fpiYGBg7kAScGJmZGg==", shards.get(1).start()); + Assert.assertEquals("/TEwMDB3IAk4MTMyNA==", shards.get(1).end()); + Assert.assertEquals("/TEwMDB3IAk4MTMyNA==", shards.get(2).start()); + Assert.assertEquals("/TEwMDB3IAk4MTMyNQ==", shards.get(2).end()); + + for (int i = 3; i < 100; i++) { + range.splitEven(i); + } + } + + @Test + public void testRangeOfRegionWithStartKey() { + byte[] start = new byte[]{-3, 0x35, 0x30, 0x30, + 0x30, 0x77, 0x4e, -37, + 0x31, 0x31, 0x30, 0x30, + 0x30, 0x37, 0x36, 0x33}; + Range range = new Range(start, END); + + List shards = range.splitEven(0); + Assert.assertEquals(1, shards.size()); + Assert.assertEquals("/TUwMDB3TtsxMTAwMDc2Mw==", shards.get(0).start()); + Assert.assertEquals(Strings.EMPTY, shards.get(0).end()); + + shards = range.splitEven(1); + Assert.assertEquals(1, shards.size()); + Assert.assertEquals("/TUwMDB3TtsxMTAwMDc2Mw==", shards.get(0).start()); + Assert.assertEquals(Strings.EMPTY, shards.get(0).end()); + + shards = range.splitEven(2); + Assert.assertEquals(2, shards.size()); + Assert.assertEquals("/TUwMDB3TtsxMTAwMDc2Mw==", shards.get(0).start()); + Assert.assertEquals("/pqYGBg7p22YmJgYGBubGQ==", shards.get(0).end()); + Assert.assertEquals("/pqYGBg7p22YmJgYGBubGQ==", shards.get(1).start()); + Assert.assertEquals(Strings.EMPTY, shards.get(1).end()); + + for (int i = 3; i < 100; i++) { + range.splitEven(i); + } + } + + @Test + public void testRangeOfRegionWithStartKeyAndEndKey() { + byte[] start = new byte[]{-3, 0x31, 0x30, 0x30, 0x30, 0x77, + 0x20, 0x09, 0x38, 0x31, 0x33, 0x32, 0x35}; + byte[] end = new byte[]{-3, 0x31, 0x33, 0x35, 0x33, 0x32, + 0x37, 0x34, 0x31, 0x35, 0x32}; + + Range range = new Range(start, end); + + List shards = range.splitEven(0); + Assert.assertEquals(1, shards.size()); + Assert.assertEquals("/TEwMDB3IAk4MTMyNQ==", shards.get(0).start()); + Assert.assertEquals("/TEzNTMyNzQxNTI=", shards.get(0).end()); + + shards = range.splitEven(1); + Assert.assertEquals(1, shards.size()); + Assert.assertEquals("/TEwMDB3IAk4MTMyNQ==", shards.get(0).start()); + Assert.assertEquals("/TEzNTMyNzQxNTI=", shards.get(0).end()); + + shards = range.splitEven(2); + Assert.assertEquals(3, shards.size()); + Assert.assertEquals("/TEwMDB3IAk4MTMyNQ==", shards.get(0).start()); + Assert.assertEquals("/TExsrHUq560szKZGg==", shards.get(0).end()); + Assert.assertEquals("/TExsrHUq560szKZGg==", shards.get(1).start()); + Assert.assertEquals("/TEzNTMyNzQxNTH//w==", shards.get(1).end()); + Assert.assertEquals("/TEzNTMyNzQxNTH//w==", shards.get(2).start()); + Assert.assertEquals("/TEzNTMyNzQxNTI=", shards.get(2).end()); + + for (int i = 3; i < 100; i++) { + range.splitEven(i); + } + } +} From c36d768efed49d15f6e34a4dec276b7652e2107d Mon Sep 17 00:00:00 2001 From: zhangyi51 Date: Wed, 23 Dec 2020 21:59:45 +0800 Subject: [PATCH 3/3] use in-order map to store regions info Change-Id: I7cbdaccf8a79227bcac2bdb953e11faf2e6da69b --- .../com/baidu/hugegraph/backend/store/hbase/HbaseTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java index 22d7b4c243..013e44a77a 100644 --- a/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java +++ b/hugegraph-hbase/src/main/java/com/baidu/hugegraph/backend/store/hbase/HbaseTable.java @@ -344,7 +344,7 @@ private static Map regionSizes(Session session, private static Map regionRanges(Session session, String namespace, String table) { - Map regionRanges = new HashMap<>(); + Map regionRanges = InsertionOrderUtil.newMap(); TableName tableName = TableName.valueOf(namespace, table); try (Admin admin = session.hbase().getAdmin()) { for (RegionInfo regionInfo : admin.getRegions(tableName)) {