Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(java): grow buffer to interger.max #1582

Merged
merged 5 commits into from
Apr 27, 2024
Merged

fix(java): grow buffer to interger.max #1582

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5e04e66
grow buffer to interger.max
chaokunyang Apr 27, 2024
62d3ab9
lint code
chaokunyang Apr 27, 2024
90e2cb3
make growBuffer in a separate method
chaokunyang Apr 27, 2024
b1cbf81
lint code
chaokunyang Apr 27, 2024
2499a29
reuse buffer threshold
chaokunyang Apr 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions java/fury-core/src/main/java/org/apache/fury/io/FuryInputStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,10 @@ public int fillBuffer(int minFillSize) {
private static byte[] growBuffer(int minFillSize, MemoryBuffer buffer) {
int newSize;
int targetSize = buffer.size() + minFillSize;
if (targetSize < BUFFER_GROW_STEP_THRESHOLD) {
newSize = targetSize << 2;
} else {
newSize = (int) (targetSize * 1.5);
}
newSize =
targetSize < BUFFER_GROW_STEP_THRESHOLD
? targetSize << 2
: (int) Math.min(targetSize * 1.5d, Integer.MAX_VALUE);
byte[] newBuffer = new byte[newSize];
byte[] heapMemory = buffer.getHeapMemory();
System.arraycopy(heapMemory, 0, newBuffer, 0, buffer.size());
Expand Down
4 changes: 3 additions & 1 deletion java/fury-core/src/main/java/org/apache/fury/io/FuryReadableChannel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,9 @@ public int fillBuffer(int minFillSize) {
int newLimit = position + minFillSize;
if (newLimit > byteBuf.capacity()) {
int newSize =
newLimit < BUFFER_GROW_STEP_THRESHOLD ? newLimit << 2 : (int) (newLimit * 1.5);
newLimit < BUFFER_GROW_STEP_THRESHOLD
? newLimit << 2
: (int) Math.min(newLimit * 1.5d, Integer.MAX_VALUE);
ByteBuffer newByteBuf = ByteBuffer.allocateDirect(newSize);
byteBuf.position(0);
newByteBuf.put(byteBuf);
Expand Down
17 changes: 13 additions & 4 deletions java/fury-core/src/main/java/org/apache/fury/memory/MemoryBuffer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1204,18 +1204,27 @@ public void writePrimitiveArray(Object arr, int offset, int numBytes) {

/** For off-heap buffer, this will make a heap buffer internally. */
public void grow(int neededSize) {
ensure(writerIndex + neededSize);
int length = writerIndex + neededSize;
if (length > size) {
growBuffer(length);
}
}

/** For off-heap buffer, this will make a heap buffer internally. */
public void ensure(int length) {
if (length > size) {
byte[] data = new byte[length * 2];
copyToUnsafe(0, data, Platform.BYTE_ARRAY_OFFSET, size());
initHeapBuffer(data, 0, data.length);
growBuffer(length);
}
}

private void growBuffer(int length) {
int newSize =
length < 104857600 ? length << 2 : (int) Math.min(length * 1.5d, Integer.MAX_VALUE);
chaokunyang marked this conversation as resolved.
Show resolved Hide resolved
byte[] data = new byte[newSize];
copyToUnsafe(0, data, Platform.BYTE_ARRAY_OFFSET, size());
initHeapBuffer(data, 0, data.length);
}

// -------------------------------------------------------------------------
// Read Methods
// -------------------------------------------------------------------------
Expand Down
Loading