From df921e915df6d44e753cb0d501b6a2cc835fdf2d Mon Sep 17 00:00:00 2001 From: V_Galaxy Date: Sat, 10 Aug 2024 17:02:30 +0800 Subject: [PATCH] refact(server): enlarge bytes write limit & remove param `big` when encode/decode string id length (#2622) As title, change limit: - vid max to 16KB - eid max to 64kb (128k as backup) - property max to 10MB (keep consistent) fix #1593 #2291 --------- Co-authored-by: imbajin --- .asf.yaml | 4 +- .../hugegraph/api/filter/AccessLogFilter.java | 3 +- .../apache/hugegraph/core/GraphManager.java | 6 +- .../hugegraph/backend/cache/OffheapCache.java | 32 +++-- .../org/apache/hugegraph/backend/id/Id.java | 3 +- .../backend/serializer/BinarySerializer.java | 4 +- .../backend/serializer/BytesBuffer.java | 110 +++++++----------- .../apache/hugegraph/structure/HugeEdge.java | 4 +- .../hugegraph/structure/HugeElement.java | 3 +- .../org/apache/hugegraph/task/HugeTask.java | 2 +- .../apache/hugegraph/core/EdgeCoreTest.java | 6 +- .../apache/hugegraph/core/VertexCoreTest.java | 7 +- .../hugegraph/unit/cache/CacheTest.java | 7 +- .../apache/hugegraph/unit/id/IdUtilTest.java | 18 +++ .../unit/serializer/BytesBufferTest.java | 55 +++++---- 15 files changed, 133 insertions(+), 131 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 6c78530425..93063dc7d0 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -34,8 +34,8 @@ github: protected_branches: master: required_status_checks: - # strict means "Require branches to be up-to-date before merging". (TODO: turnoff when branch is stable) - strict: true + # strict means "Require PR to be up-to-date before merging". (enable when branch unstable) + strict: false # contexts are the names of checks that must pass (now only enable the basic check) contexts: - Analyze (java) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java index d429db4d9b..ae53686467 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.net.URI; -import org.apache.hugegraph.auth.HugeAuthenticator; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.config.ServerOptions; import org.apache.hugegraph.core.GraphManager; @@ -124,7 +123,7 @@ public void filter(ContainerRequestContext requestContext, GraphManager manager = managerProvider.get(); // TODO: transfer Authorizer if we need after. if (manager.requireAuthentication()) { - manager.unauthorize(requestContext.getSecurityContext()); + manager.unauthorized(requestContext.getSecurityContext()); } } diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java index 37939c2019..5cba0a0f0f 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java @@ -265,7 +265,7 @@ public HugeAuthenticator.User authenticate(Map credentials) return this.authenticator().authenticate(credentials); } - public void unauthorize(SecurityContext context) { + public void unauthorized(SecurityContext context) { this.authenticator().unauthorize(context); } @@ -515,7 +515,7 @@ private boolean supportRoleElection() { private void addMetrics(HugeConfig config) { final MetricManager metric = MetricManager.INSTANCE; - // Force to add server reporter + // Force to add a server reporter ServerReporter reporter = ServerReporter.instance(metric.getRegistry()); reporter.start(60L, TimeUnit.SECONDS); @@ -610,7 +610,7 @@ private void dropGraph(HugeGraph graph) { /* * Will fill graph instance into HugeFactory.graphs after - * GraphFactory.open() succeed, remove it when graph drop + * GraphFactory.open() succeed, remove it when the graph drops */ HugeFactory.remove(graph); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java index d641c8276f..7ed4efcd66 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java @@ -55,14 +55,18 @@ public class OffheapCache extends AbstractCache { private final AbstractSerializer serializer; public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes) { - // NOTE: capacity unit is bytes, the super capacity expect elements size + this(graph, capacity, avgEntryBytes, Runtime.getRuntime().availableProcessors() * 2); + } + + public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes, int segments) { + // NOTE: capacity unit is bytes, the super capacity expects elements' size super(capacity); - long capacityInBytes = capacity * (avgEntryBytes + 64L); + long capacityInBytes = Math.max(capacity, segments) * (avgEntryBytes + 64L); if (capacityInBytes <= 0L) { capacityInBytes = 1L; } this.graph = graph; - this.cache = this.builder().capacity(capacityInBytes).build(); + this.cache = this.builder().capacity(capacityInBytes).segmentCount(segments).build(); this.serializer = new BinarySerializer(); } @@ -162,19 +166,18 @@ private class IdSerializer implements CacheSerializer { @Override public Id deserialize(ByteBuffer input) { - return BytesBuffer.wrap(input).readId(true); + return BytesBuffer.wrap(input).readId(); } @Override public void serialize(Id id, ByteBuffer output) { - BytesBuffer.wrap(output).writeId(id, true); + BytesBuffer.wrap(output).writeId(id); } @Override public int serializedSize(Id id) { // NOTE: return size must be == actual bytes to write - return BytesBuffer.allocate(id.length() + 2) - .writeId(id, true).position(); + return BytesBuffer.allocate(id.length() + 2).writeId(id).position(); } } @@ -229,7 +232,6 @@ public ByteBuffer asBuffer() { } BytesBuffer buffer = BytesBuffer.allocate(64 * listSize); - // May fail to serialize and throw exception here this.serialize(this.value, buffer); @@ -276,8 +278,7 @@ private Object deserialize(BytesBuffer buffer) { } } - private void serializeList(BytesBuffer buffer, - Collection list) { + private void serializeList(BytesBuffer buffer, Collection list) { // Write list buffer.writeVInt(list.size()); for (Object i : list) { @@ -295,8 +296,7 @@ private List deserializeList(BytesBuffer buffer) { return list; } - private void serializeElement(BytesBuffer buffer, - ValueType type, Object value) { + private void serializeElement(BytesBuffer buffer, ValueType type, Object value) { E.checkNotNull(value, "serialize value"); BackendEntry entry; if (type == ValueType.VERTEX) { @@ -332,14 +332,12 @@ private Object deserializeElement(ValueType type, BytesBuffer buffer) { } private HugeException unsupported(ValueType type) { - throw new HugeException( - "Unsupported deserialize type: %s", type); + throw new HugeException("Unsupported deserialize type: %s", type); } private HugeException unsupported(Object value) { - throw new HugeException( - "Unsupported type of serialize value: '%s'(%s)", - value, value.getClass()); + throw new HugeException("Unsupported type of serialize value: '%s'(%s)", + value, value.getClass()); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java index b0e5613495..077aa43dbb 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java @@ -67,8 +67,7 @@ public char prefix() { } public static IdType valueOfPrefix(String id) { - E.checkArgument(id != null && !id.isEmpty(), - "Invalid id '%s'", id); + E.checkArgument(id != null && !id.isEmpty(), "Invalid id '%s'", id); switch (id.charAt(0)) { case 'L': return IdType.LONG; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java index 37a7e9a9ca..0dd4c9d3ee 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java @@ -1312,9 +1312,9 @@ private Id readId(byte[] value) { } private byte[] writeIds(Collection ids) { - E.checkState(ids.size() <= BytesBuffer.UINT16_MAX, + E.checkState(ids.size() <= BytesBuffer.MAX_PROPERTIES, "The number of properties of vertex/edge label " + - "can't exceed '%s'", BytesBuffer.UINT16_MAX); + "can't exceed '%s'", BytesBuffer.MAX_PROPERTIES); int size = 2; for (Id id : ids) { size += (1 + id.length()); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java index b99c2c8b5e..c13d611082 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java @@ -51,22 +51,27 @@ public final class BytesBuffer extends OutputStream { public static final int CHAR_LEN = Character.BYTES; public static final int FLOAT_LEN = Float.BYTES; public static final int DOUBLE_LEN = Double.BYTES; - public static final int BLOB_LEN = 4; + public static final int BYTES_LEN = 4; + public static final int BLOB_LEN = 5; public static final int UINT8_MAX = ((byte) -1) & 0xff; public static final int UINT16_MAX = ((short) -1) & 0xffff; public static final long UINT32_MAX = (-1) & 0xffffffffL; + // TODO: support user-defined configuration // NOTE: +1 to let code 0 represent length 1 - public static final int ID_LEN_MASK = 0x7f; - public static final int ID_LEN_MAX = 0x7f + 1; // 128 - public static final int BIG_ID_LEN_MAX = 0x7fff + 1; // 32768 + public static final int ID_LEN_MAX = 0x3fff + 1; // 16KB + public static final int EID_LEN_MAX = 64 * 1024; public static final byte STRING_ENDING_BYTE = (byte) 0x00; public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff; - public static final int STRING_LEN_MAX = UINT16_MAX; + + // TODO: support user-defined configuration + public static final long BYTES_LEN_MAX = 10 * Bytes.MB; public static final long BLOB_LEN_MAX = 1 * Bytes.GB; + public static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX; + // The value must be in range [8, ID_LEN_MAX] public static final int INDEX_HASH_ID_THRESHOLD = 32; @@ -87,8 +92,7 @@ public BytesBuffer() { public BytesBuffer(int capacity) { E.checkArgument(capacity <= MAX_BUFFER_CAPACITY, - "Capacity exceeds max buffer capacity: %s", - MAX_BUFFER_CAPACITY); + "Capacity exceeds max buffer capacity: %s", MAX_BUFFER_CAPACITY); this.buffer = ByteBuffer.allocate(capacity); this.resize = true; } @@ -167,8 +171,7 @@ private void require(int size) { // Extra capacity as buffer int newCapacity = size + this.buffer.limit() + DEFAULT_CAPACITY; E.checkArgument(newCapacity <= MAX_BUFFER_CAPACITY, - "Capacity exceeds max buffer capacity: %s", - MAX_BUFFER_CAPACITY); + "Capacity exceeds max buffer capacity: %s", MAX_BUFFER_CAPACITY); ByteBuffer newBuffer = ByteBuffer.allocate(newCapacity); this.buffer.flip(); newBuffer.put(this.buffer); @@ -288,10 +291,9 @@ public double readDouble() { } public BytesBuffer writeBytes(byte[] bytes) { - E.checkArgument(bytes.length <= UINT16_MAX, - "The max length of bytes is %s, but got %s", - UINT16_MAX, bytes.length); - require(SHORT_LEN + bytes.length); + E.checkArgument(bytes.length <= BYTES_LEN_MAX, "The max length of bytes is %s, but got %s", + BYTES_LEN_MAX, bytes.length); + require(BYTES_LEN + bytes.length); this.writeVInt(bytes.length); this.write(bytes); return this; @@ -304,8 +306,8 @@ public byte[] readBytes() { } public BytesBuffer writeBigBytes(byte[] bytes) { - E.checkArgument(bytes.length <= BLOB_LEN_MAX, - "The max length of bytes is %s, but got %s", + // TODO: note the max blob size should be 128MB (due to MAX_BUFFER_CAPACITY) + E.checkArgument(bytes.length <= BLOB_LEN_MAX, "The max length of bytes is %s, but got %s", BLOB_LEN_MAX, bytes.length); require(BLOB_LEN + bytes.length); this.writeVInt(bytes.length); @@ -346,9 +348,7 @@ public BytesBuffer writeStringWithEnding(String value) { assert !Bytes.contains(bytes, STRING_ENDING_BYTE_FF) : "Invalid UTF8 bytes: " + value; if (Bytes.contains(bytes, STRING_ENDING_BYTE)) { - E.checkArgument(false, - "Can't contains byte '0x00' in string: '%s'", - value); + E.checkArgument(false, "Can't contains byte '0x00' in string: '%s'", value); } this.write(bytes); } @@ -430,8 +430,7 @@ public BytesBuffer writeVInt(int value) { public int readVInt() { byte leading = this.read(); - E.checkArgument(leading != 0x80, - "Unexpected varint with leading byte '0x%s'", + E.checkArgument(leading != 0x80, "Unexpected varint with leading byte '0x%s'", Bytes.toHex(leading)); int value = leading & 0x7f; if (leading >= 0) { @@ -450,8 +449,7 @@ public int readVInt() { } } - E.checkArgument(i < 5, - "Unexpected varint %s with too many bytes(%s)", + E.checkArgument(i < 5, "Unexpected varint %s with too many bytes(%s)", value, i + 1); E.checkArgument(i < 4 || (leading & 0x70) == 0, "Unexpected varint %s with leading byte '0x%s'", @@ -494,8 +492,7 @@ public BytesBuffer writeVLong(long value) { public long readVLong() { byte leading = this.read(); - E.checkArgument(leading != 0x80, - "Unexpected varlong with leading byte '0x%s'", + E.checkArgument(leading != 0x80, "Unexpected varlong with leading byte '0x%s'", Bytes.toHex(leading)); long value = leading & 0x7fL; if (leading >= 0) { @@ -514,8 +511,7 @@ public long readVLong() { } } - E.checkArgument(i < 10, - "Unexpected varlong %s with too many bytes(%s)", + E.checkArgument(i < 10, "Unexpected varlong %s with too many bytes(%s)", value, i + 1); E.checkArgument(i < 9 || (leading & 0x7e) == 0, "Unexpected varlong %s with leading byte '0x%s'", @@ -581,8 +577,7 @@ public void writeProperty(DataType dataType, Object value) { this.writeString((String) value); break; case BLOB: - byte[] bytes = value instanceof byte[] ? - (byte[]) value : ((Blob) value).bytes(); + byte[] bytes = value instanceof byte[] ? (byte[]) value : ((Blob) value).bytes(); this.writeBigBytes(bytes); break; case UUID: @@ -592,6 +587,7 @@ public void writeProperty(DataType dataType, Object value) { this.writeLong(uuid.getLeastSignificantBits()); break; default: + // TODO: replace Kryo with Fury (https://github.com/apache/fury) this.writeBytes(KryoUtil.toKryoWithType(value)); break; } @@ -624,15 +620,12 @@ public Object readProperty(DataType dataType) { case UUID: return new UUID(this.readLong(), this.readLong()); default: + // TODO: replace Kryo with Fury (https://github.com/apache/fury) return KryoUtil.fromKryoWithType(this.readBytes()); } } public BytesBuffer writeId(Id id) { - return this.writeId(id, false); - } - - public BytesBuffer writeId(Id id, boolean big) { switch (id.type()) { case LONG: // Number Id @@ -640,36 +633,33 @@ public BytesBuffer writeId(Id id, boolean big) { this.writeNumber(value); break; case UUID: - // UUID Id + // UUID ID byte[] bytes = id.asBytes(); assert bytes.length == Id.UUID_LENGTH; this.writeUInt8(0x7f); // 0b01111111 means UUID this.write(bytes); break; case EDGE: - // Edge Id + // Edge ID this.writeUInt8(0x7e); // 0b01111110 means EdgeId this.writeEdgeId(id); break; default: - // String Id + // String Id (VertexID) bytes = id.asBytes(); int len = bytes.length; E.checkArgument(len > 0, "Can't write empty id"); - if (!big) { - E.checkArgument(len <= ID_LEN_MAX, - "Id max length is %s, but got %s {%s}", - ID_LEN_MAX, len, id); - len -= 1; // mapping [1, 128] to [0, 127] + E.checkArgument(len <= ID_LEN_MAX, "Big id max length is %s, but got %s {%s}", + ID_LEN_MAX, len, id); + len -= 1; // mapping [1, 16384] to [0, 16383] + if (len <= 0x3f) { + // If length is <= 63, use a single byte with the highest bit set to 1 this.writeUInt8(len | 0x80); } else { - E.checkArgument(len <= BIG_ID_LEN_MAX, - "Big id max length is %s, but got %s {%s}", - BIG_ID_LEN_MAX, len, id); - len -= 1; int high = len >> 8; int low = len & 0xff; - this.writeUInt8(high | 0x80); + // Write high 8 bits with highest two bits set to 11 + this.writeUInt8(high | 0xc0); this.writeUInt8(low); } this.write(bytes); @@ -679,10 +669,6 @@ public BytesBuffer writeId(Id id, boolean big) { } public Id readId() { - return this.readId(false); - } - - public Id readId(boolean big) { byte b = this.read(); boolean number = (b & 0x80) == 0; if (number) { @@ -698,13 +684,13 @@ public Id readId(boolean big) { } } else { // String Id - int len = b & ID_LEN_MASK; - if (big) { + int len = b & 0x3f; // Take the lowest 6 bits as part of the length + if ((b & 0x40) != 0) { // If the 7th bit is set, length information spans 2 bytes int high = len << 8; int low = this.readUInt8(); len = high + low; } - len += 1; // restore [0, 127] to [1, 128] + len += 1; // restore [0, 16383] to [1, 16384] byte[] id = this.read(len); return IdGenerator.of(id, IdType.STRING); } @@ -739,12 +725,11 @@ public BytesBuffer writeIndexId(Id id, HugeType type, boolean withEnding) { this.write(bytes); if (type.isStringIndex()) { if (Bytes.contains(bytes, STRING_ENDING_BYTE)) { - // Not allow STRING_ENDING_BYTE exist in string index id + // Not allow STRING_ENDING_BYTE to exist in string index id E.checkArgument(false, "The %s type index id can't contains " + - "byte '0x%s', but got: 0x%s", type, - Bytes.toHex(STRING_ENDING_BYTE), - Bytes.toHex(bytes)); + "byte '0x%s', but got: 0x%s", + type, Bytes.toHex(STRING_ENDING_BYTE), Bytes.toHex(bytes)); } if (withEnding) { this.writeStringWithEnding(""); @@ -806,12 +791,9 @@ public BinaryId parseOlapId(HugeType type, boolean isOlap) { } // Parse id from bytes int start = this.buffer.position(); - /** - * OLAP - * {PropertyKey}{VertexId} - */ + // OLAP {PropertyKey}{VertexId} if (isOlap) { - // First read olap property id + // Read olap property id first Id pkId = this.readId(); } Id id = this.readId(); @@ -890,8 +872,7 @@ private void writeNumber(long val) { } private long readNumber(byte b) { - E.checkArgument((b & 0x80) == 0, - "Not a number type with prefix byte '0x%s'", + E.checkArgument((b & 0x80) == 0, "Not a number type with prefix byte '0x%s'", Bytes.toHex(b)); // Parse the kind from byte 0kkksxxx int kind = b >>> 4; @@ -948,8 +929,7 @@ private byte[] readBytesWithEnding() { break; } } - E.checkArgument(foundEnding, "Not found ending '0x%s'", - Bytes.toHex(STRING_ENDING_BYTE)); + E.checkArgument(foundEnding, "Not found ending '0x%s'", Bytes.toHex(STRING_ENDING_BYTE)); int end = this.buffer.position() - 1; int len = end - start; if (len <= 0) { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java index 53b8a54933..f38c2b067f 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java @@ -147,9 +147,9 @@ public void assignId() { if (this.fresh()) { int len = this.id.length(); - E.checkArgument(len <= BytesBuffer.BIG_ID_LEN_MAX, + E.checkArgument(len <= BytesBuffer.EID_LEN_MAX, "The max length of edge id is %s, but got %s {%s}", - BytesBuffer.BIG_ID_LEN_MAX, len, this.id); + BytesBuffer.EID_LEN_MAX, len, this.id); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java index 137e623d86..adb12f31bc 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java @@ -55,7 +55,6 @@ public abstract class HugeElement implements Element, GraphType, Idfiable, Compa private static final MutableIntObjectMap> EMPTY_MAP = CollectionFactory.newIntObjectMap(); - private static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX; private final HugeGraph graph; private MutableIntObjectMap> properties; @@ -279,7 +278,7 @@ public HugeProperty setProperty(HugeProperty prop) { PropertyKey pkey = prop.propertyKey(); E.checkArgument(this.properties.containsKey(intFromId(pkey.id())) || - this.properties.size() < MAX_PROPERTIES, + this.properties.size() < BytesBuffer.MAX_PROPERTIES, "Exceeded the maximum number of properties"); return this.properties.put(intFromId(pkey.id()), prop); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java index c07b9e8d47..f9e4f120f4 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java @@ -725,7 +725,7 @@ private void checkPropertySize(String property, String propertyName) { } private void checkPropertySize(int propertyLength, String propertyName) { - long propertyLimit = BytesBuffer.STRING_LEN_MAX; + long propertyLimit = BytesBuffer.MAX_PROPERTIES; HugeGraph graph = this.scheduler().graph(); if (propertyName.equals(P.INPUT)) { propertyLimit = graph.option(CoreOptions.TASK_INPUT_SIZE_LIMIT); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java index b79cae1963..265d408742 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java @@ -75,7 +75,6 @@ import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; import org.junit.Assume; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import com.google.common.collect.ImmutableList; @@ -470,12 +469,13 @@ public void testAddEdgeWithLargeSortkey() { Vertex book = graph.addVertex(T.label, "book", "name", "Test-Book-1"); Assert.assertThrows(IllegalArgumentException.class, () -> { - final int LEN = BytesBuffer.BIG_ID_LEN_MAX; + final int LEN = BytesBuffer.EID_LEN_MAX; String largeTime = "{large-time}" + new String(new byte[LEN]); james.addEdge("write", book, "time", largeTime); graph.tx().commit(); }, e -> { - Assert.assertContains("The max length of edge id is 32768", + Assert.assertContains(String.format("The max length of edge id is %s", + BytesBuffer.EID_LEN_MAX), e.getMessage()); }); } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index c9a83ddc15..a329de3afb 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -44,6 +44,7 @@ import org.apache.hugegraph.backend.query.Condition; import org.apache.hugegraph.backend.query.ConditionQuery; import org.apache.hugegraph.backend.query.Query; +import org.apache.hugegraph.backend.serializer.BytesBuffer; import org.apache.hugegraph.backend.store.BackendTable; import org.apache.hugegraph.backend.store.Shard; import org.apache.hugegraph.backend.tx.GraphTransaction; @@ -1039,10 +1040,10 @@ public void testAddVertexWithCustomizeStringIdStrategyWithoutValidId() { "name", "marko", "age", 18, "city", "Beijing"); }); - // Expect id length <= 128 + // Expect id length <= BytesBuffer.ID_LEN_MAX Assert.assertThrows(IllegalArgumentException.class, () -> { - String largeId = new String(new byte[128]) + "."; - assert largeId.length() == 129; + String largeId = new String(new byte[BytesBuffer.ID_LEN_MAX]) + "."; + assert largeId.length() == BytesBuffer.ID_LEN_MAX + 1; graph.addVertex(T.label, "programmer", T.id, largeId, "name", "marko", "age", 18, "city", "Beijing"); }); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java index 63837c8aab..d9bd4cf953 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java @@ -139,6 +139,7 @@ protected void checkNotInCache(Cache cache, Id id) { public static class OffheapCacheTest extends CacheTest { private static final long ENTRY_SIZE = 40L; + private static final int SEGMENTS = 4; private final HugeGraph graph = Mockito.mock(HugeGraph.class); @Override @@ -148,7 +149,7 @@ protected Cache newCache() { @Override protected Cache newCache(long capacity) { - return new OffheapCache(this.graph(), capacity, ENTRY_SIZE); + return new OffheapCache(this.graph(), capacity, ENTRY_SIZE, SEGMENTS); } @Override @@ -325,7 +326,7 @@ public void testUpdateAndGetWithDataType() { @Test public void testUpdateAndGetWithSameSizeAndCapacity() { - int limit = 40; + int limit = 1000; Cache cache = newCache(limit); Map map = new LimitMap(limit); @@ -451,7 +452,7 @@ public void testSize() { @Test public void testSizeWithReachCapacity() { - int limit = 20; + int limit = 1000; Cache cache = newCache(limit); Map map = new LimitMap(limit); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java index fa582322ab..fe9925b2ba 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java @@ -154,4 +154,22 @@ private byte[] genBytes(String string) { } return bytes; } + + /** + * Converts a byte array to a hexadecimal string. + * + * @param bytes the byte array to convert + * @return the hexadecimal string representation of the byte array + */ + private String bytesToHex(byte[] bytes) { + StringBuilder hexString = new StringBuilder(); + for (byte b : bytes) { + String hex = Integer.toHexString(0xFF & b); + if (hex.length() == 1) { + hexString.append('0'); // pad with leading zero if needed + } + hexString.append(hex); + } + return hexString.toString(); + } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java index 9072652908..fb25a5b255 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java @@ -96,29 +96,35 @@ public void testStringId() { @Test public void testStringIdWithBigSize() { Id id = IdGenerator.of(genString(127)); - byte[] bytes = genBytes(128); - bytes[0] = (byte) 0xfe; + byte[] bytes = genBytes(129); + bytes[0] = (byte) 0xc0; + bytes[1] = (byte) 0x7e; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) .writeId(id).bytes()); Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); id = IdGenerator.of(genString(128)); - bytes = genBytes(129); - bytes[0] = (byte) 0xff; + bytes = genBytes(130); + bytes[0] = (byte) 0xc0; + bytes[1] = (byte) 0x7f; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) .writeId(id).bytes()); Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); Assert.assertThrows(IllegalArgumentException.class, () -> { - BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(129))); + BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX + 1))); }, e -> { - Assert.assertContains("Id max length is 128, but got 129", + Assert.assertContains(String.format("Big id max length is %s, but got %s", + BytesBuffer.ID_LEN_MAX, + BytesBuffer.ID_LEN_MAX + 1), e.getMessage()); }); Assert.assertThrows(IllegalArgumentException.class, () -> { - BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(130))); + BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX + 2))); }, e -> { - Assert.assertContains("Id max length is 128, but got 130", + Assert.assertContains(String.format("Big id max length is %s, but got %s", + BytesBuffer.ID_LEN_MAX, + BytesBuffer.ID_LEN_MAX + 2), e.getMessage()); }); } @@ -127,33 +133,34 @@ public void testStringIdWithBigSize() { public void testStringBigId() { Id id = IdGenerator.of(genString(128)); byte[] bytes = genBytes(130); - bytes[0] = (byte) 0x80; + bytes[0] = (byte) 0xc0; bytes[1] = (byte) 0x7f; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) - .writeId(id, true).bytes()); - Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId(true)); + .writeId(id).bytes()); + Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); - id = IdGenerator.of(genString(32512)); - bytes = genBytes(32514); - bytes[0] = (byte) 0xfe; - bytes[1] = (byte) 0xff; + id = IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX - 1)); + bytes = genBytes(BytesBuffer.ID_LEN_MAX + 1); + bytes[0] = (byte) 0xff; + bytes[1] = (byte) 0xfe; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) - .writeId(id, true).bytes()); - Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId(true)); + .writeId(id).bytes()); + Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); - id = IdGenerator.of(genString(32768)); - bytes = genBytes(32770); + id = IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX)); + bytes = genBytes(BytesBuffer.ID_LEN_MAX + 2); bytes[0] = (byte) 0xff; bytes[1] = (byte) 0xff; Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0) - .writeId(id, true).bytes()); - Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId(true)); + .writeId(id).bytes()); + Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId()); Assert.assertThrows(IllegalArgumentException.class, () -> { - BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(32769)), - true); + BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX + 1))); }, e -> { - Assert.assertContains("Big id max length is 32768, but got 32769", + Assert.assertContains(String.format("Big id max length is %s, but got %s", + BytesBuffer.ID_LEN_MAX, + BytesBuffer.ID_LEN_MAX + 1), e.getMessage()); }); }