refact(server): enlarge bytes write limit & remove param big when e…

…ncode/decode string id length (#2622) As title, change limit: - vid max to 16KB - eid max to 64kb (128k as backup) - property max to 10MB (keep consistent) fix #1593 #2291 --------- Co-authored-by: imbajin <[email protected]>
apache · Aug 10, 2024 · df921e9 · df921e9
1 parent 7df736a
commit df921e9
Show file tree

Hide file tree

Showing 15 changed files with 133 additions and 131 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -34,8 +34,8 @@ github:
   protected_branches:
     master:
       required_status_checks:
-        # strict means "Require branches to be up-to-date before merging". (TODO: turnoff when branch is stable)
-        strict: true
+        # strict means "Require PR to be up-to-date before merging". (enable when branch unstable)
+        strict: false
         # contexts are the names of checks that must pass (now only enable the basic check)
         contexts:
           - Analyze (java)

diff --git a/...h-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java b/...h-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java
@@ -26,7 +26,6 @@
 import java.io.IOException;
 import java.net.URI;
 
-import org.apache.hugegraph.auth.HugeAuthenticator;
 import org.apache.hugegraph.config.HugeConfig;
 import org.apache.hugegraph.config.ServerOptions;
 import org.apache.hugegraph.core.GraphManager;
@@ -124,7 +123,7 @@ public void filter(ContainerRequestContext requestContext,
         GraphManager manager = managerProvider.get();
         // TODO: transfer Authorizer if we need after.
         if (manager.requireAuthentication()) {
-            manager.unauthorize(requestContext.getSecurityContext());
+            manager.unauthorized(requestContext.getSecurityContext());
         }
     }
 

diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java
@@ -265,7 +265,7 @@ public HugeAuthenticator.User authenticate(Map<String, String> credentials)
         return this.authenticator().authenticate(credentials);
     }
 
-    public void unauthorize(SecurityContext context) {
+    public void unauthorized(SecurityContext context) {
         this.authenticator().unauthorize(context);
     }
 
@@ -515,7 +515,7 @@ private boolean supportRoleElection() {
 
     private void addMetrics(HugeConfig config) {
         final MetricManager metric = MetricManager.INSTANCE;
-        // Force to add server reporter
+        // Force to add a server reporter
         ServerReporter reporter = ServerReporter.instance(metric.getRegistry());
         reporter.start(60L, TimeUnit.SECONDS);
 
@@ -610,7 +610,7 @@ private void dropGraph(HugeGraph graph) {
 
         /*
          * Will fill graph instance into HugeFactory.graphs after
-         * GraphFactory.open() succeed, remove it when graph drop
+         * GraphFactory.open() succeed, remove it when the graph drops
          */
         HugeFactory.remove(graph);
     }

diff --git a/...-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java b/...-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java
@@ -55,14 +55,18 @@ public class OffheapCache extends AbstractCache<Id, Object> {
     private final AbstractSerializer serializer;
 
     public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes) {
-        // NOTE: capacity unit is bytes, the super capacity expect elements size
+        this(graph, capacity, avgEntryBytes, Runtime.getRuntime().availableProcessors() * 2);
+    }
+
+    public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes, int segments) {
+        // NOTE: capacity unit is bytes, the super capacity expects elements' size
         super(capacity);
-        long capacityInBytes = capacity * (avgEntryBytes + 64L);
+        long capacityInBytes = Math.max(capacity, segments) * (avgEntryBytes + 64L);
         if (capacityInBytes <= 0L) {
             capacityInBytes = 1L;
         }
         this.graph = graph;
-        this.cache = this.builder().capacity(capacityInBytes).build();
+        this.cache = this.builder().capacity(capacityInBytes).segmentCount(segments).build();
         this.serializer = new BinarySerializer();
     }
 
@@ -162,19 +166,18 @@ private class IdSerializer implements CacheSerializer<Id> {
 
         @Override
         public Id deserialize(ByteBuffer input) {
-            return BytesBuffer.wrap(input).readId(true);
+            return BytesBuffer.wrap(input).readId();
         }
 
         @Override
         public void serialize(Id id, ByteBuffer output) {
-            BytesBuffer.wrap(output).writeId(id, true);
+            BytesBuffer.wrap(output).writeId(id);
         }
 
         @Override
         public int serializedSize(Id id) {
             // NOTE: return size must be == actual bytes to write
-            return BytesBuffer.allocate(id.length() + 2)
-                              .writeId(id, true).position();
+            return BytesBuffer.allocate(id.length() + 2).writeId(id).position();
         }
     }
 
@@ -229,7 +232,6 @@ public ByteBuffer asBuffer() {
                 }
 
                 BytesBuffer buffer = BytesBuffer.allocate(64 * listSize);
-
                 // May fail to serialize and throw exception here
                 this.serialize(this.value, buffer);
 
@@ -276,8 +278,7 @@ private Object deserialize(BytesBuffer buffer) {
             }
         }
 
-        private void serializeList(BytesBuffer buffer,
-                                   Collection<Object> list) {
+        private void serializeList(BytesBuffer buffer, Collection<Object> list) {
             // Write list
             buffer.writeVInt(list.size());
             for (Object i : list) {
@@ -295,8 +296,7 @@ private List<Object> deserializeList(BytesBuffer buffer) {
             return list;
         }
 
-        private void serializeElement(BytesBuffer buffer,
-                                      ValueType type, Object value) {
+        private void serializeElement(BytesBuffer buffer, ValueType type, Object value) {
             E.checkNotNull(value, "serialize value");
             BackendEntry entry;
             if (type == ValueType.VERTEX) {
@@ -332,14 +332,12 @@ private Object deserializeElement(ValueType type, BytesBuffer buffer) {
         }
 
         private HugeException unsupported(ValueType type) {
-            throw new HugeException(
-                    "Unsupported deserialize type: %s", type);
+            throw new HugeException("Unsupported deserialize type: %s", type);
         }
 
         private HugeException unsupported(Object value) {
-            throw new HugeException(
-                    "Unsupported type of serialize value: '%s'(%s)",
-                    value, value.getClass());
+            throw new HugeException("Unsupported type of serialize value: '%s'(%s)",
+                                    value, value.getClass());
         }
     }
 

diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java
@@ -67,8 +67,7 @@ public char prefix() {
         }
 
         public static IdType valueOfPrefix(String id) {
-            E.checkArgument(id != null && !id.isEmpty(),
-                            "Invalid id '%s'", id);
+            E.checkArgument(id != null && !id.isEmpty(), "Invalid id '%s'", id);
             switch (id.charAt(0)) {
                 case 'L':
                     return IdType.LONG;

diff --git a/...ugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java b/...ugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java
@@ -1312,9 +1312,9 @@ private Id readId(byte[] value) {
         }
 
         private byte[] writeIds(Collection<Id> ids) {
-            E.checkState(ids.size() <= BytesBuffer.UINT16_MAX,
+            E.checkState(ids.size() <= BytesBuffer.MAX_PROPERTIES,
                          "The number of properties of vertex/edge label " +
-                         "can't exceed '%s'", BytesBuffer.UINT16_MAX);
+                         "can't exceed '%s'", BytesBuffer.MAX_PROPERTIES);
             int size = 2;
             for (Id id : ids) {
                 size += (1 + id.length());