Skip to content

Commit

Permalink
fix(java): grow buffer to interger.max (apache#1582)
Browse files Browse the repository at this point in the history
## What does this PR do?

grow buffer to interger.max

## Related issues

apache#1576 


## Does this PR introduce any user-facing change?

<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/incubator-fury/issues/new/choose)
describing the need to do so and update the document if necessary.
-->

- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?


## Benchmark

<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
  • Loading branch information
chaokunyang authored Apr 27, 2024
1 parent e664615 commit 73f9e9b
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,10 @@ public int fillBuffer(int minFillSize) {
private static byte[] growBuffer(int minFillSize, MemoryBuffer buffer) {
int newSize;
int targetSize = buffer.size() + minFillSize;
if (targetSize < BUFFER_GROW_STEP_THRESHOLD) {
newSize = targetSize << 2;
} else {
newSize = (int) (targetSize * 1.5);
}
newSize =
targetSize < MemoryBuffer.BUFFER_GROW_STEP_THRESHOLD
? targetSize << 2
: (int) Math.min(targetSize * 1.5d, Integer.MAX_VALUE);
byte[] newBuffer = new byte[newSize];
byte[] heapMemory = buffer.getHeapMemory();
System.arraycopy(heapMemory, 0, newBuffer, 0, buffer.size());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,9 @@ public int fillBuffer(int minFillSize) {
int newLimit = position + minFillSize;
if (newLimit > byteBuf.capacity()) {
int newSize =
newLimit < BUFFER_GROW_STEP_THRESHOLD ? newLimit << 2 : (int) (newLimit * 1.5);
newLimit < MemoryBuffer.BUFFER_GROW_STEP_THRESHOLD
? newLimit << 2
: (int) Math.min(newLimit * 1.5d, Integer.MAX_VALUE);
ByteBuffer newByteBuf = ByteBuffer.allocateDirect(newSize);
byteBuf.position(0);
newByteBuf.put(byteBuf);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@

/** A streaming reader to make {@link MemoryBuffer} to support streaming reading. */
public interface FuryStreamReader {
int BUFFER_GROW_STEP_THRESHOLD = 100 * 1024 * 1024;

/**
* Read stream and fill the data to underlying {@link MemoryBuffer}, which is also the buffer
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
* DesiredMethodLimit,MaxRecursiveInlineLevel,FreqInlineSize,MaxInlineSize
*/
public final class MemoryBuffer {
public static final int BUFFER_GROW_STEP_THRESHOLD = 100 * 1024 * 1024;
private static final Unsafe UNSAFE = Platform.UNSAFE;
private static final boolean LITTLE_ENDIAN = (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN);

Expand Down Expand Up @@ -1204,18 +1205,29 @@ public void writePrimitiveArray(Object arr, int offset, int numBytes) {

/** For off-heap buffer, this will make a heap buffer internally. */
public void grow(int neededSize) {
ensure(writerIndex + neededSize);
int length = writerIndex + neededSize;
if (length > size) {
growBuffer(length);
}
}

/** For off-heap buffer, this will make a heap buffer internally. */
public void ensure(int length) {
if (length > size) {
byte[] data = new byte[length * 2];
copyToUnsafe(0, data, Platform.BYTE_ARRAY_OFFSET, size());
initHeapBuffer(data, 0, data.length);
growBuffer(length);
}
}

private void growBuffer(int length) {
int newSize =
length < BUFFER_GROW_STEP_THRESHOLD
? length << 2
: (int) Math.min(length * 1.5d, Integer.MAX_VALUE);
byte[] data = new byte[newSize];
copyToUnsafe(0, data, Platform.BYTE_ARRAY_OFFSET, size());
initHeapBuffer(data, 0, data.length);
}

// -------------------------------------------------------------------------
// Read Methods
// -------------------------------------------------------------------------
Expand Down

0 comments on commit 73f9e9b

Please sign in to comment.