Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refact(server): enlarge bytes write limit & remove param big when encode/decode string id length #2622

Merged
merged 20 commits into from
Aug 10, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1312,9 +1312,9 @@ private Id readId(byte[] value) {
}

private byte[] writeIds(Collection<Id> ids) {
E.checkState(ids.size() <= BytesBuffer.UINT16_MAX,
E.checkState(ids.size() <= BytesBuffer.MAX_PROPERTIES,
"The number of properties of vertex/edge label " +
"can't exceed '%s'", BytesBuffer.UINT16_MAX);
"can't exceed '%s'", BytesBuffer.MAX_PROPERTIES);
int size = 2;
for (Id id : ids) {
size += (1 + id.length());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,20 +52,27 @@ public final class BytesBuffer extends OutputStream {
public static final int FLOAT_LEN = Float.BYTES;
public static final int DOUBLE_LEN = Double.BYTES;
public static final int BLOB_LEN = 4;
public static final int BYTES_LEN = 5;

public static final int UINT8_MAX = ((byte) -1) & 0xff;
public static final int UINT16_MAX = ((short) -1) & 0xffff;
public static final long UINT32_MAX = (-1) & 0xffffffffL;
public static final int INT32_MAX = Integer.MAX_VALUE;

// TODO: support user-defined configuration
// NOTE: +1 to let code 0 represent length 1
public static final int ID_LEN_MASK = 0x7f;
public static final int ID_LEN_MAX = 0x7f + 1; // 128
public static final int BIG_ID_LEN_MAX = 0x7fff + 1; // 32768

public static final byte STRING_ENDING_BYTE = (byte) 0x00;
public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff;
public static final int STRING_LEN_MAX = UINT16_MAX;

// TODO: support user-defined configuration
public static final long BLOB_LEN_MAX = 1 * Bytes.GB;
public static final long BYTES_LEN_MAX = INT32_MAX;

public static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX;

// The value must be in range [8, ID_LEN_MAX]
public static final int INDEX_HASH_ID_THRESHOLD = 32;
Expand Down Expand Up @@ -288,10 +295,10 @@ public double readDouble() {
}

public BytesBuffer writeBytes(byte[] bytes) {
E.checkArgument(bytes.length <= UINT16_MAX,
E.checkArgument(bytes.length <= BYTES_LEN_MAX,
VGalaxies marked this conversation as resolved.
Show resolved Hide resolved
"The max length of bytes is %s, but got %s",
UINT16_MAX, bytes.length);
require(SHORT_LEN + bytes.length);
BYTES_LEN_MAX, bytes.length);
require(BYTES_LEN + bytes.length);
this.writeVInt(bytes.length);
this.write(bytes);
return this;
Expand Down Expand Up @@ -629,7 +636,7 @@ public Object readProperty(DataType dataType) {
}

public BytesBuffer writeId(Id id) {
return this.writeId(id, false);
return this.writeId(id, true);
}

public BytesBuffer writeId(Id id, boolean big) {
Expand Down Expand Up @@ -679,7 +686,7 @@ public BytesBuffer writeId(Id id, boolean big) {
}

public Id readId() {
return this.readId(false);
return this.readId(true);
}

public Id readId(boolean big) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ public abstract class HugeElement implements Element, GraphType, Idfiable, Compa

private static final MutableIntObjectMap<HugeProperty<?>> EMPTY_MAP =
CollectionFactory.newIntObjectMap();
private static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX;

private final HugeGraph graph;
private MutableIntObjectMap<HugeProperty<?>> properties;
Expand Down Expand Up @@ -279,7 +278,7 @@ public <V> HugeProperty<?> setProperty(HugeProperty<V> prop) {
PropertyKey pkey = prop.propertyKey();

E.checkArgument(this.properties.containsKey(intFromId(pkey.id())) ||
this.properties.size() < MAX_PROPERTIES,
this.properties.size() < BytesBuffer.MAX_PROPERTIES,
"Exceeded the maximum number of properties");
return this.properties.put(intFromId(pkey.id()), prop);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -725,7 +725,7 @@ private void checkPropertySize(String property, String propertyName) {
}

private void checkPropertySize(int propertyLength, String propertyName) {
long propertyLimit = BytesBuffer.STRING_LEN_MAX;
long propertyLimit = BytesBuffer.MAX_PROPERTIES;
HugeGraph graph = this.scheduler().graph();
if (propertyName.equals(P.INPUT)) {
propertyLimit = graph.option(CoreOptions.TASK_INPUT_SIZE_LIMIT);
Expand Down
Loading