Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refact(server): enlarge bytes write limit & remove param big when encode/decode string id length #2622

Merged
merged 20 commits into from
Aug 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .asf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ github:
protected_branches:
master:
required_status_checks:
# strict means "Require branches to be up-to-date before merging". (TODO: turnoff when branch is stable)
strict: true
# strict means "Require PR to be up-to-date before merging". (enable when branch unstable)
strict: false
# contexts are the names of checks that must pass (now only enable the basic check)
contexts:
- Analyze (java)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import java.io.IOException;
import java.net.URI;

import org.apache.hugegraph.auth.HugeAuthenticator;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.config.ServerOptions;
import org.apache.hugegraph.core.GraphManager;
Expand Down Expand Up @@ -124,7 +123,7 @@ public void filter(ContainerRequestContext requestContext,
GraphManager manager = managerProvider.get();
// TODO: transfer Authorizer if we need after.
if (manager.requireAuthentication()) {
manager.unauthorize(requestContext.getSecurityContext());
manager.unauthorized(requestContext.getSecurityContext());
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ public HugeAuthenticator.User authenticate(Map<String, String> credentials)
return this.authenticator().authenticate(credentials);
}

public void unauthorize(SecurityContext context) {
public void unauthorized(SecurityContext context) {
this.authenticator().unauthorize(context);
}

Expand Down Expand Up @@ -515,7 +515,7 @@ private boolean supportRoleElection() {

private void addMetrics(HugeConfig config) {
final MetricManager metric = MetricManager.INSTANCE;
// Force to add server reporter
// Force to add a server reporter
ServerReporter reporter = ServerReporter.instance(metric.getRegistry());
reporter.start(60L, TimeUnit.SECONDS);

Expand Down Expand Up @@ -610,7 +610,7 @@ private void dropGraph(HugeGraph graph) {

/*
* Will fill graph instance into HugeFactory.graphs after
* GraphFactory.open() succeed, remove it when graph drop
* GraphFactory.open() succeed, remove it when the graph drops
*/
HugeFactory.remove(graph);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,18 @@
private final AbstractSerializer serializer;

public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes) {
// NOTE: capacity unit is bytes, the super capacity expect elements size
this(graph, capacity, avgEntryBytes, Runtime.getRuntime().availableProcessors() * 2);
}

public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes, int segments) {
// NOTE: capacity unit is bytes, the super capacity expects elements' size
super(capacity);
long capacityInBytes = capacity * (avgEntryBytes + 64L);
long capacityInBytes = Math.max(capacity, segments) * (avgEntryBytes + 64L);
if (capacityInBytes <= 0L) {
capacityInBytes = 1L;
}
this.graph = graph;
this.cache = this.builder().capacity(capacityInBytes).build();
this.cache = this.builder().capacity(capacityInBytes).segmentCount(segments).build();
this.serializer = new BinarySerializer();
}

Expand Down Expand Up @@ -162,19 +166,18 @@

@Override
public Id deserialize(ByteBuffer input) {
return BytesBuffer.wrap(input).readId(true);
return BytesBuffer.wrap(input).readId();

Check warning on line 169 in hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java

View check run for this annotation

Codecov / codecov/patch

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java#L169

Added line #L169 was not covered by tests
}

@Override
public void serialize(Id id, ByteBuffer output) {
BytesBuffer.wrap(output).writeId(id, true);
BytesBuffer.wrap(output).writeId(id);
}

@Override
public int serializedSize(Id id) {
// NOTE: return size must be == actual bytes to write
return BytesBuffer.allocate(id.length() + 2)
.writeId(id, true).position();
return BytesBuffer.allocate(id.length() + 2).writeId(id).position();
}
}

Expand Down Expand Up @@ -229,7 +232,6 @@
}

BytesBuffer buffer = BytesBuffer.allocate(64 * listSize);

// May fail to serialize and throw exception here
this.serialize(this.value, buffer);

Expand Down Expand Up @@ -276,8 +278,7 @@
}
}

private void serializeList(BytesBuffer buffer,
Collection<Object> list) {
private void serializeList(BytesBuffer buffer, Collection<Object> list) {
// Write list
buffer.writeVInt(list.size());
for (Object i : list) {
Expand All @@ -295,8 +296,7 @@
return list;
}

private void serializeElement(BytesBuffer buffer,
ValueType type, Object value) {
private void serializeElement(BytesBuffer buffer, ValueType type, Object value) {
E.checkNotNull(value, "serialize value");
BackendEntry entry;
if (type == ValueType.VERTEX) {
Expand Down Expand Up @@ -332,14 +332,12 @@
}

private HugeException unsupported(ValueType type) {
throw new HugeException(
"Unsupported deserialize type: %s", type);
throw new HugeException("Unsupported deserialize type: %s", type);

Check warning on line 335 in hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java

View check run for this annotation

Codecov / codecov/patch

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java#L335

Added line #L335 was not covered by tests
}

private HugeException unsupported(Object value) {
throw new HugeException(
"Unsupported type of serialize value: '%s'(%s)",
value, value.getClass());
throw new HugeException("Unsupported type of serialize value: '%s'(%s)",
value, value.getClass());
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,7 @@ public char prefix() {
}

public static IdType valueOfPrefix(String id) {
E.checkArgument(id != null && !id.isEmpty(),
"Invalid id '%s'", id);
E.checkArgument(id != null && !id.isEmpty(), "Invalid id '%s'", id);
switch (id.charAt(0)) {
case 'L':
return IdType.LONG;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1312,9 +1312,9 @@ private Id readId(byte[] value) {
}

private byte[] writeIds(Collection<Id> ids) {
E.checkState(ids.size() <= BytesBuffer.UINT16_MAX,
E.checkState(ids.size() <= BytesBuffer.MAX_PROPERTIES,
"The number of properties of vertex/edge label " +
"can't exceed '%s'", BytesBuffer.UINT16_MAX);
"can't exceed '%s'", BytesBuffer.MAX_PROPERTIES);
int size = 2;
for (Id id : ids) {
size += (1 + id.length());
Expand Down
Loading
Loading