From fa8ebdebd1109217a6ac1ae2f173e1ab9d400d82 Mon Sep 17 00:00:00 2001 From: Laurent Goujon Date: Mon, 29 Apr 2024 16:05:41 -0700 Subject: [PATCH] GH-43479: [Java] Change visibility of MemoryUtil.UNSAFE `MemoryUtil.UNSAFE` field is a public field which provides unrestricted access to `sun.misc.Unsafe` instance which may cause misusage and possibly JVM crashes. Make the field private and only allow indirect use of Unsafe through `MemoryUtil` methods --- .../adapter/jdbc/consumer/ClobConsumer.java | 16 +--- .../FixedWidthOutOfPlaceVectorSorter.java | 2 +- .../VariableWidthOutOfPlaceVectorSorter.java | 2 +- .../apache/arrow/c/ArrowArrayUtilityTest.java | 12 +-- .../org/apache/arrow/memory/ArrowBuf.java | 82 +++++++++---------- .../memory/util/ByteFunctionHelpers.java | 42 +++++----- .../apache/arrow/memory/util/MemoryUtil.java | 72 +++++++++++++++- .../arrow/memory/util/hash/MurmurHasher.java | 4 +- .../arrow/memory/util/hash/SimpleHasher.java | 6 +- .../DefaultAllocationManagerFactory.java | 6 +- .../arrow/memory/TestBaseAllocator.java | 25 +----- .../arrow/memory/TestForeignAllocation.java | 4 +- .../unsafe/UnsafeAllocationManager.java | 6 +- .../arrow/vector/BaseFixedWidthVector.java | 2 +- .../apache/arrow/vector/BitVectorHelper.java | 13 ++- .../apache/arrow/vector/Decimal256Vector.java | 44 +++++----- .../apache/arrow/vector/DecimalVector.java | 43 ++++------ .../arrow/vector/util/DecimalUtility.java | 8 +- .../arrow/vector/util/VectorAppender.java | 18 ++-- .../arrow/vector/TestBitVectorHelper.java | 12 +-- 20 files changed, 218 insertions(+), 201 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java index 7deba1cbffebd..9fcdd42414dfa 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java @@ -86,12 +86,8 @@ public void consume(ResultSet resultSet) throws SQLException { while ((dataBuffer.writerIndex() + bytes.length) > dataBuffer.capacity()) { vector.reallocDataBuffer(); } - MemoryUtil.UNSAFE.copyMemory( - bytes, - MemoryUtil.BYTE_ARRAY_BASE_OFFSET, - null, - dataBuffer.memoryAddress() + startIndex + totalBytes, - bytes.length); + MemoryUtil.copyToMemory( + bytes, 0, dataBuffer.memoryAddress() + startIndex + totalBytes, bytes.length); totalBytes += bytes.length; read += readSize; @@ -133,12 +129,8 @@ public void consume(ResultSet resultSet) throws SQLException { while ((dataBuffer.writerIndex() + bytes.length) > dataBuffer.capacity()) { vector.reallocDataBuffer(); } - MemoryUtil.UNSAFE.copyMemory( - bytes, - MemoryUtil.BYTE_ARRAY_BASE_OFFSET, - null, - dataBuffer.memoryAddress() + startIndex + totalBytes, - bytes.length); + MemoryUtil.copyToMemory( + bytes, 0, dataBuffer.memoryAddress() + startIndex + totalBytes, bytes.length); totalBytes += bytes.length; read += readSize; diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java index 817e890a5abe1..ac8b5a4be56aa 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java @@ -77,7 +77,7 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator co BitVectorHelper.unsetBit(dstValidityBuffer, dstIndex); } else { BitVectorHelper.setBit(dstValidityBuffer, dstIndex); - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( srcValueBuffer.memoryAddress() + srcIndex * ((long) valueWidth), dstValueBuffer.memoryAddress() + dstIndex * ((long) valueWidth), valueWidth); diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java index 8f58dc0dcee0f..a3aca83441d2f 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java @@ -91,7 +91,7 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator co int valueLength = srcOffsetBuffer.getInt((srcIndex + 1) * ((long) BaseVariableWidthVector.OFFSET_WIDTH)) - srcOffset; - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( srcValueBuffer.memoryAddress() + srcOffset, dstValueBuffer.memoryAddress() + dstOffset, valueLength); diff --git a/java/c/src/test/java/org/apache/arrow/c/ArrowArrayUtilityTest.java b/java/c/src/test/java/org/apache/arrow/c/ArrowArrayUtilityTest.java index 46f09ae5f0e8f..1d4cb411fab45 100644 --- a/java/c/src/test/java/org/apache/arrow/c/ArrowArrayUtilityTest.java +++ b/java/c/src/test/java/org/apache/arrow/c/ArrowArrayUtilityTest.java @@ -103,14 +103,14 @@ allocator, dummyHandle, new ArrowFieldNode(/* length= */ 0, 0), new long[] {0})) @Test void cleanupAfterFailure() throws Exception { // Note values are all dummy values here - long address = MemoryUtil.UNSAFE.allocateMemory(16); + long address = MemoryUtil.allocateMemory(16); try (BufferImportTypeVisitor visitor = new BufferImportTypeVisitor( allocator, dummyHandle, new ArrowFieldNode(0, 0), new long[] {address})) { // This fails, but only after we've already imported a buffer. assertThrows(IllegalStateException.class, () -> visitor.visit(new ArrowType.Int(32, true))); } finally { - MemoryUtil.UNSAFE.freeMemory(address); + MemoryUtil.freeMemory(address); } } @@ -119,7 +119,7 @@ void bufferAssociatedWithAllocator() throws Exception { // Note values are all dummy values here final long bufferSize = 16; final long fieldLength = bufferSize / IntVector.TYPE_WIDTH; - long address = MemoryUtil.UNSAFE.allocateMemory(bufferSize); + long address = MemoryUtil.allocateMemory(bufferSize); long baseline = allocator.getAllocatedMemory(); ArrowFieldNode fieldNode = new ArrowFieldNode(fieldLength, 0); try (BufferImportTypeVisitor visitor = @@ -134,7 +134,7 @@ void bufferAssociatedWithAllocator() throws Exception { .isEqualTo(allocator); assertThat(allocator.getAllocatedMemory()).isEqualTo(baseline + bufferSize); } finally { - MemoryUtil.UNSAFE.freeMemory(address); + MemoryUtil.freeMemory(address); } assertThat(allocator.getAllocatedMemory()).isEqualTo(baseline); } @@ -161,7 +161,7 @@ void releaseRetain() { @Test void associate() { final long bufferSize = 16; - final long address = MemoryUtil.UNSAFE.allocateMemory(bufferSize); + final long address = MemoryUtil.allocateMemory(bufferSize); try { ArrowArray array = ArrowArray.allocateNew(allocator); ReferenceCountedArrowArray handle = new ReferenceCountedArrowArray(array); @@ -173,7 +173,7 @@ void associate() { handle.release(); assertThat(array.isClosed()).isTrue(); } finally { - MemoryUtil.UNSAFE.freeMemory(address); + MemoryUtil.freeMemory(address); } } } diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java index 24a0ea0761ec0..a958092a5789a 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java @@ -310,7 +310,7 @@ private void checkIndexD(long index, long fieldLength) { */ public long getLong(long index) { chk(index, LONG_SIZE); - return MemoryUtil.UNSAFE.getLong(addr(index)); + return MemoryUtil.getLong(addr(index)); } /** @@ -322,7 +322,7 @@ public long getLong(long index) { */ public void setLong(long index, long value) { chk(index, LONG_SIZE); - MemoryUtil.UNSAFE.putLong(addr(index), value); + MemoryUtil.putLong(addr(index), value); } /** @@ -345,7 +345,7 @@ public float getFloat(long index) { */ public void setFloat(long index, float value) { chk(index, FLOAT_SIZE); - MemoryUtil.UNSAFE.putInt(addr(index), Float.floatToRawIntBits(value)); + MemoryUtil.putInt(addr(index), Float.floatToRawIntBits(value)); } /** @@ -368,7 +368,7 @@ public double getDouble(long index) { */ public void setDouble(long index, double value) { chk(index, DOUBLE_SIZE); - MemoryUtil.UNSAFE.putLong(addr(index), Double.doubleToRawLongBits(value)); + MemoryUtil.putLong(addr(index), Double.doubleToRawLongBits(value)); } /** @@ -391,7 +391,7 @@ public char getChar(long index) { */ public void setChar(long index, int value) { chk(index, SHORT_SIZE); - MemoryUtil.UNSAFE.putShort(addr(index), (short) value); + MemoryUtil.putShort(addr(index), (short) value); } /** @@ -403,7 +403,7 @@ public void setChar(long index, int value) { */ public int getInt(long index) { chk(index, INT_SIZE); - return MemoryUtil.UNSAFE.getInt(addr(index)); + return MemoryUtil.getInt(addr(index)); } /** @@ -414,7 +414,7 @@ public int getInt(long index) { */ public void setInt(long index, int value) { chk(index, INT_SIZE); - MemoryUtil.UNSAFE.putInt(addr(index), value); + MemoryUtil.putInt(addr(index), value); } /** @@ -426,7 +426,7 @@ public void setInt(long index, int value) { */ public short getShort(long index) { chk(index, SHORT_SIZE); - return MemoryUtil.UNSAFE.getShort(addr(index)); + return MemoryUtil.getShort(addr(index)); } /** @@ -449,7 +449,7 @@ public void setShort(long index, int value) { */ public void setShort(long index, short value) { chk(index, SHORT_SIZE); - MemoryUtil.UNSAFE.putShort(addr(index), value); + MemoryUtil.putShort(addr(index), value); } /** @@ -461,7 +461,7 @@ public void setShort(long index, short value) { */ public void setByte(long index, int value) { chk(index, 1); - MemoryUtil.UNSAFE.putByte(addr(index), (byte) value); + MemoryUtil.putByte(addr(index), (byte) value); } /** @@ -473,7 +473,7 @@ public void setByte(long index, int value) { */ public void setByte(long index, byte value) { chk(index, 1); - MemoryUtil.UNSAFE.putByte(addr(index), value); + MemoryUtil.putByte(addr(index), value); } /** @@ -485,7 +485,7 @@ public void setByte(long index, byte value) { */ public byte getByte(long index) { chk(index, 1); - return MemoryUtil.UNSAFE.getByte(addr(index)); + return MemoryUtil.getByte(addr(index)); } /*--------------------------------------------------* @@ -566,7 +566,7 @@ public void readBytes(byte[] dst) { */ public void writeByte(byte value) { ensureWritable(1); - MemoryUtil.UNSAFE.putByte(addr(writerIndex), value); + MemoryUtil.putByte(addr(writerIndex), value); ++writerIndex; } @@ -577,7 +577,7 @@ public void writeByte(byte value) { */ public void writeByte(int value) { ensureWritable(1); - MemoryUtil.UNSAFE.putByte(addr(writerIndex), (byte) value); + MemoryUtil.putByte(addr(writerIndex), (byte) value); ++writerIndex; } @@ -612,7 +612,7 @@ public void writeBytes(byte[] src, int srcIndex, int length) { */ public void writeShort(int value) { ensureWritable(SHORT_SIZE); - MemoryUtil.UNSAFE.putShort(addr(writerIndex), (short) value); + MemoryUtil.putShort(addr(writerIndex), (short) value); writerIndex += SHORT_SIZE; } @@ -623,7 +623,7 @@ public void writeShort(int value) { */ public void writeInt(int value) { ensureWritable(INT_SIZE); - MemoryUtil.UNSAFE.putInt(addr(writerIndex), value); + MemoryUtil.putInt(addr(writerIndex), value); writerIndex += INT_SIZE; } @@ -634,7 +634,7 @@ public void writeInt(int value) { */ public void writeLong(long value) { ensureWritable(LONG_SIZE); - MemoryUtil.UNSAFE.putLong(addr(writerIndex), value); + MemoryUtil.putLong(addr(writerIndex), value); writerIndex += LONG_SIZE; } @@ -645,7 +645,7 @@ public void writeLong(long value) { */ public void writeFloat(float value) { ensureWritable(FLOAT_SIZE); - MemoryUtil.UNSAFE.putInt(addr(writerIndex), Float.floatToRawIntBits(value)); + MemoryUtil.putInt(addr(writerIndex), Float.floatToRawIntBits(value)); writerIndex += FLOAT_SIZE; } @@ -656,7 +656,7 @@ public void writeFloat(float value) { */ public void writeDouble(double value) { ensureWritable(DOUBLE_SIZE); - MemoryUtil.UNSAFE.putLong(addr(writerIndex), Double.doubleToRawLongBits(value)); + MemoryUtil.putLong(addr(writerIndex), Double.doubleToRawLongBits(value)); writerIndex += DOUBLE_SIZE; } @@ -727,8 +727,7 @@ public void getBytes(long index, byte[] dst, int dstIndex, int length) { if (length != 0) { // copy "length" bytes from this ArrowBuf starting at addr(index) address // into dst byte array at dstIndex onwards - MemoryUtil.copyMemory( - null, addr(index), dst, MemoryUtil.BYTE_ARRAY_BASE_OFFSET + dstIndex, length); + MemoryUtil.copyFromMemory(addr(index), dst, dstIndex, length); } } @@ -766,8 +765,7 @@ public void setBytes(long index, byte[] src, int srcIndex, long length) { if (length > 0) { // copy "length" bytes from src byte array at the starting index (srcIndex) // into this ArrowBuf starting at address "addr(index)" - MemoryUtil.copyMemory( - src, MemoryUtil.BYTE_ARRAY_BASE_OFFSET + srcIndex, null, addr(index), length); + MemoryUtil.copyToMemory(src, srcIndex, addr(index), length); } } @@ -792,7 +790,7 @@ public void getBytes(long index, ByteBuffer dst) { // at address srcAddress into the dst ByteBuffer starting at // address dstAddress final long dstAddress = MemoryUtil.getByteBufferAddress(dst) + dst.position(); - MemoryUtil.copyMemory(null, srcAddress, null, dstAddress, dst.remaining()); + MemoryUtil.copyMemory(srcAddress, dstAddress, dst.remaining()); // after copy, bump the next write position for the dst ByteBuffer dst.position(dst.position() + dst.remaining()); } else if (dst.hasArray()) { @@ -800,12 +798,7 @@ public void getBytes(long index, ByteBuffer dst) { // at address srcAddress into the dst ByteBuffer starting at // index dstIndex final int dstIndex = dst.arrayOffset() + dst.position(); - MemoryUtil.copyMemory( - null, - srcAddress, - dst.array(), - MemoryUtil.BYTE_ARRAY_BASE_OFFSET + dstIndex, - dst.remaining()); + MemoryUtil.copyFromMemory(srcAddress, dst.array(), dstIndex, dst.remaining()); // after copy, bump the next write position for the dst ByteBuffer dst.position(dst.position() + dst.remaining()); } else { @@ -834,15 +827,14 @@ public void setBytes(long index, ByteBuffer src) { // copy src.remaining() bytes of data from src ByteBuffer starting at // address srcAddress into this ArrowBuf starting at address dstAddress final long srcAddress = MemoryUtil.getByteBufferAddress(src) + src.position(); - MemoryUtil.copyMemory(null, srcAddress, null, dstAddress, length); + MemoryUtil.copyMemory(srcAddress, dstAddress, length); // after copy, bump the next read position for the src ByteBuffer src.position(src.position() + length); } else if (src.hasArray()) { // copy src.remaining() bytes of data from src ByteBuffer starting at // index srcIndex into this ArrowBuf starting at address dstAddress final int srcIndex = src.arrayOffset() + src.position(); - MemoryUtil.copyMemory( - src.array(), MemoryUtil.BYTE_ARRAY_BASE_OFFSET + srcIndex, null, dstAddress, length); + MemoryUtil.copyToMemory(src.array(), srcIndex, dstAddress, length); // after copy, bump the next read position for the src ByteBuffer src.position(src.position() + length); } else { @@ -852,19 +844,19 @@ public void setBytes(long index, ByteBuffer src) { // copy word at a time while (length - 128 >= LONG_SIZE) { for (int x = 0; x < 16; x++) { - MemoryUtil.UNSAFE.putLong(dstAddress, src.getLong()); + MemoryUtil.putLong(dstAddress, src.getLong()); length -= LONG_SIZE; dstAddress += LONG_SIZE; } } while (length >= LONG_SIZE) { - MemoryUtil.UNSAFE.putLong(dstAddress, src.getLong()); + MemoryUtil.putLong(dstAddress, src.getLong()); length -= LONG_SIZE; dstAddress += LONG_SIZE; } // copy last byte while (length > 0) { - MemoryUtil.UNSAFE.putByte(dstAddress, src.get()); + MemoryUtil.putByte(dstAddress, src.get()); --length; ++dstAddress; } @@ -892,7 +884,7 @@ public void setBytes(long index, ByteBuffer src, int srcIndex, int length) { // srcAddress into this ArrowBuf at address dstAddress final long srcAddress = MemoryUtil.getByteBufferAddress(src) + srcIndex; final long dstAddress = addr(index); - MemoryUtil.copyMemory(null, srcAddress, null, dstAddress, length); + MemoryUtil.copyMemory(srcAddress, dstAddress, length); } else { if (srcIndex == 0 && src.capacity() == length) { // copy the entire ByteBuffer from start to end of length @@ -932,7 +924,7 @@ public void getBytes(long index, ArrowBuf dst, long dstIndex, int length) { // dstAddress final long srcAddress = addr(index); final long dstAddress = dst.memoryAddress() + (long) dstIndex; - MemoryUtil.copyMemory(null, srcAddress, null, dstAddress, length); + MemoryUtil.copyMemory(srcAddress, dstAddress, length); } } @@ -962,7 +954,7 @@ public void setBytes(long index, ArrowBuf src, long srcIndex, long length) { // dstAddress final long srcAddress = src.memoryAddress() + srcIndex; final long dstAddress = addr(index); - MemoryUtil.copyMemory(null, srcAddress, null, dstAddress, length); + MemoryUtil.copyMemory(srcAddress, dstAddress, length); } } @@ -982,7 +974,7 @@ public void setBytes(long index, ArrowBuf src) { checkIndex(index, length); final long srcAddress = src.memoryAddress() + src.readerIndex; final long dstAddress = addr(index); - MemoryUtil.copyMemory(null, srcAddress, null, dstAddress, length); + MemoryUtil.copyMemory(srcAddress, dstAddress, length); src.readerIndex(src.readerIndex + length); } @@ -1007,7 +999,7 @@ public int setBytes(long index, InputStream in, int length) throws IOException { if (readBytes > 0) { // copy readBytes length of data from the tmp byte array starting // at srcIndex 0 into this ArrowBuf starting at address addr(index) - MemoryUtil.copyMemory(tmp, MemoryUtil.BYTE_ARRAY_BASE_OFFSET, null, addr(index), readBytes); + MemoryUtil.copyToMemory(tmp, 0, addr(index), readBytes); } } return readBytes; @@ -1029,7 +1021,7 @@ public void getBytes(long index, OutputStream out, int length) throws IOExceptio // copy length bytes of data from this ArrowBuf starting at // address addr(index) into the tmp byte array starting at index 0 byte[] tmp = new byte[length]; - MemoryUtil.copyMemory(null, addr(index), tmp, MemoryUtil.BYTE_ARRAY_BASE_OFFSET, length); + MemoryUtil.copyFromMemory(addr(index), tmp, 0, length); // write the copied data to output stream out.write(tmp); } @@ -1173,7 +1165,7 @@ public ArrowBuf writerIndex(long writerIndex) { public ArrowBuf setZero(long index, long length) { if (length != 0) { this.checkIndex(index, length); - MemoryUtil.UNSAFE.setMemory(this.addr + index, length, (byte) 0); + MemoryUtil.setMemory(this.addr + index, length, (byte) 0); } return this; } @@ -1191,7 +1183,7 @@ public ArrowBuf setZero(long index, long length) { public ArrowBuf setOne(int index, int length) { if (length != 0) { this.checkIndex(index, length); - MemoryUtil.UNSAFE.setMemory(this.addr + index, length, (byte) 0xff); + MemoryUtil.setMemory(this.addr + index, length, (byte) 0xff); } return this; } @@ -1207,7 +1199,7 @@ public ArrowBuf setOne(int index, int length) { public ArrowBuf setOne(long index, long length) { if (length != 0) { this.checkIndex(index, length); - MemoryUtil.UNSAFE.setMemory(this.addr + index, length, (byte) 0xff); + MemoryUtil.setMemory(this.addr + index, length, (byte) 0xff); } return this; } diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java index 44289183a318d..9243be399b6db 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java @@ -61,8 +61,8 @@ private static int memEqual( while (n > 63) { for (int x = 0; x < 8; x++) { - long leftLong = MemoryUtil.UNSAFE.getLong(lPos); - long rightLong = MemoryUtil.UNSAFE.getLong(rPos); + long leftLong = MemoryUtil.getLong(lPos); + long rightLong = MemoryUtil.getLong(rPos); if (leftLong != rightLong) { return 0; } @@ -73,8 +73,8 @@ private static int memEqual( } while (n > 7) { - long leftLong = MemoryUtil.UNSAFE.getLong(lPos); - long rightLong = MemoryUtil.UNSAFE.getLong(rPos); + long leftLong = MemoryUtil.getLong(lPos); + long rightLong = MemoryUtil.getLong(rPos); if (leftLong != rightLong) { return 0; } @@ -84,8 +84,8 @@ private static int memEqual( } if (n > 3) { - int leftInt = MemoryUtil.UNSAFE.getInt(lPos); - int rightInt = MemoryUtil.UNSAFE.getInt(rPos); + int leftInt = MemoryUtil.getInt(lPos); + int rightInt = MemoryUtil.getInt(rPos); if (leftInt != rightInt) { return 0; } @@ -95,8 +95,8 @@ private static int memEqual( } while (n-- != 0) { - byte leftByte = MemoryUtil.UNSAFE.getByte(lPos); - byte rightByte = MemoryUtil.UNSAFE.getByte(rPos); + byte leftByte = MemoryUtil.getByte(lPos); + byte rightByte = MemoryUtil.getByte(rPos); if (leftByte != rightByte) { return 0; } @@ -141,8 +141,8 @@ private static int memcmp( while (n > 63) { for (int x = 0; x < 8; x++) { - long leftLong = MemoryUtil.UNSAFE.getLong(lPos); - long rightLong = MemoryUtil.UNSAFE.getLong(rPos); + long leftLong = MemoryUtil.getLong(lPos); + long rightLong = MemoryUtil.getLong(rPos); if (leftLong != rightLong) { if (LITTLE_ENDIAN) { return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong)); @@ -157,8 +157,8 @@ private static int memcmp( } while (n > 7) { - long leftLong = MemoryUtil.UNSAFE.getLong(lPos); - long rightLong = MemoryUtil.UNSAFE.getLong(rPos); + long leftLong = MemoryUtil.getLong(lPos); + long rightLong = MemoryUtil.getLong(rPos); if (leftLong != rightLong) { if (LITTLE_ENDIAN) { return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong)); @@ -172,8 +172,8 @@ private static int memcmp( } if (n > 3) { - int leftInt = MemoryUtil.UNSAFE.getInt(lPos); - int rightInt = MemoryUtil.UNSAFE.getInt(rPos); + int leftInt = MemoryUtil.getInt(lPos); + int rightInt = MemoryUtil.getInt(rPos); if (leftInt != rightInt) { if (LITTLE_ENDIAN) { return unsignedIntCompare(Integer.reverseBytes(leftInt), Integer.reverseBytes(rightInt)); @@ -187,8 +187,8 @@ private static int memcmp( } while (n-- != 0) { - byte leftByte = MemoryUtil.UNSAFE.getByte(lPos); - byte rightByte = MemoryUtil.UNSAFE.getByte(rPos); + byte leftByte = MemoryUtil.getByte(lPos); + byte rightByte = MemoryUtil.getByte(rPos); if (leftByte != rightByte) { return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1; } @@ -248,8 +248,8 @@ private static int memcmp( int rPos = rStart; while (n > 7) { - long leftLong = MemoryUtil.UNSAFE.getLong(lPos); - long rightLong = MemoryUtil.UNSAFE.getLong(right, MemoryUtil.BYTE_ARRAY_BASE_OFFSET + rPos); + long leftLong = MemoryUtil.getLong(lPos); + long rightLong = MemoryUtil.getLong(right, rPos); if (leftLong != rightLong) { if (LITTLE_ENDIAN) { return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong)); @@ -263,8 +263,8 @@ private static int memcmp( } if (n > 3) { - int leftInt = MemoryUtil.UNSAFE.getInt(lPos); - int rightInt = MemoryUtil.UNSAFE.getInt(right, MemoryUtil.BYTE_ARRAY_BASE_OFFSET + rPos); + int leftInt = MemoryUtil.getInt(lPos); + int rightInt = MemoryUtil.getInt(right, rPos); if (leftInt != rightInt) { if (LITTLE_ENDIAN) { return unsignedIntCompare(Integer.reverseBytes(leftInt), Integer.reverseBytes(rightInt)); @@ -278,7 +278,7 @@ private static int memcmp( } while (n-- != 0) { - byte leftByte = MemoryUtil.UNSAFE.getByte(lPos); + byte leftByte = MemoryUtil.getByte(lPos); byte rightByte = right[rPos]; if (leftByte != rightByte) { return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1; diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java index c1b44c3932659..acf77547fbcdd 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java @@ -33,13 +33,13 @@ public class MemoryUtil { private static final @Nullable Constructor DIRECT_BUFFER_CONSTRUCTOR; /** The unsafe object from which to access the off-heap memory. */ - public static final Unsafe UNSAFE; + private static final Unsafe UNSAFE; /** The start offset of array data relative to the start address of the array object. */ - public static final long BYTE_ARRAY_BASE_OFFSET; + private static final long BYTE_ARRAY_BASE_OFFSET; /** The offset of the address field with the {@link java.nio.ByteBuffer} object. */ - static final long BYTE_BUFFER_ADDRESS_OFFSET; + private static final long BYTE_BUFFER_ADDRESS_OFFSET; /** If the native byte order is little-endian. */ public static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN; @@ -178,7 +178,7 @@ public static ByteBuffer directBuffer(long address, int capacity) { @SuppressWarnings( "nullness:argument") // to handle null assignment on third party dependency: Unsafe - public static void copyMemory( + private static void copyMemory( @Nullable Object srcBase, long srcOffset, @Nullable Object destBase, @@ -186,4 +186,68 @@ public static void copyMemory( long bytes) { UNSAFE.copyMemory(srcBase, srcOffset, destBase, destOffset, bytes); } + + public static void copyMemory(long srcAddress, long destAddress, long bytes) { + UNSAFE.copyMemory(srcAddress, destAddress, bytes); + } + + public static void copyToMemory(byte[] src, long srcIndex, long destAddress, long bytes) { + copyMemory(src, BYTE_ARRAY_BASE_OFFSET + srcIndex, null, destAddress, bytes); + } + + public static void copyFromMemory(long srcAddress, byte[] dest, long destIndex, long bytes) { + copyMemory(null, srcAddress, dest, BYTE_ARRAY_BASE_OFFSET + destIndex, bytes); + } + + public static byte getByte(long address) { + return UNSAFE.getByte(address); + } + + public static void putByte(long address, byte value) { + UNSAFE.putByte(address, value); + } + + public static short getShort(long address) { + return UNSAFE.getShort(address); + } + + public static void putShort(long address, short value) { + UNSAFE.putShort(address, value); + } + + public static int getInt(long address) { + return UNSAFE.getInt(address); + } + + public static void putInt(long address, int value) { + UNSAFE.putInt(address, value); + } + + public static long getLong(long address) { + return UNSAFE.getLong(address); + } + + public static void putLong(long address, long value) { + UNSAFE.putLong(address, value); + } + + public static void setMemory(long address, long bytes, byte value) { + UNSAFE.setMemory(address, bytes, value); + } + + public static int getInt(byte[] bytes, int index) { + return UNSAFE.getInt(bytes, BYTE_ARRAY_BASE_OFFSET + index); + } + + public static long getLong(byte[] bytes, int index) { + return UNSAFE.getLong(bytes, BYTE_ARRAY_BASE_OFFSET + index); + } + + public static long allocateMemory(long bytes) { + return UNSAFE.allocateMemory(bytes); + } + + public static void freeMemory(long address) { + UNSAFE.freeMemory(address); + } } diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java index eaf4a833c4eeb..7907018d0a815 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java @@ -86,7 +86,7 @@ public static int hashCode(long address, long length, int seed) { int index = 0; int hash = seed; while (index + 4 <= length) { - int intValue = MemoryUtil.UNSAFE.getInt(address + index); + int intValue = MemoryUtil.getInt(address + index); hash = combineHashCode(hash, intValue); index += 4; } @@ -96,7 +96,7 @@ public static int hashCode(long address, long length, int seed) { int intValue = 0; for (long i = length - 1; i >= index; i--) { intValue <<= 8; - intValue |= (MemoryUtil.UNSAFE.getByte(address + i) & 0x000000ff); + intValue |= (MemoryUtil.getByte(address + i) & 0x000000ff); index += 1; } hash = combineHashCode(hash, intValue); diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java index b9987a5ecb049..5c1384163e81e 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java @@ -52,21 +52,21 @@ public int hashCode(long address, long length) { int hashValue = 0; int index = 0; while (index + 8 <= length) { - long longValue = MemoryUtil.UNSAFE.getLong(address + index); + long longValue = MemoryUtil.getLong(address + index); int longHash = getLongHashCode(longValue); hashValue = combineHashCode(hashValue, longHash); index += 8; } if (index + 4 <= length) { - int intValue = MemoryUtil.UNSAFE.getInt(address + index); + int intValue = MemoryUtil.getInt(address + index); int intHash = intValue; hashValue = combineHashCode(hashValue, intHash); index += 4; } while (index < length) { - byte byteValue = MemoryUtil.UNSAFE.getByte(address + index); + byte byteValue = MemoryUtil.getByte(address + index); int byteHash = byteValue; hashValue = combineHashCode(hashValue, byteHash); index += 1; diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java index 83118face8674..348ed3e7933b0 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java @@ -27,13 +27,13 @@ public class DefaultAllocationManagerFactory implements AllocationManager.Factor public static final AllocationManager.Factory FACTORY = new DefaultAllocationManagerFactory(); private static final ArrowBuf EMPTY = - new ArrowBuf(ReferenceManager.NO_OP, null, 0, MemoryUtil.UNSAFE.allocateMemory(0)); + new ArrowBuf(ReferenceManager.NO_OP, null, 0, MemoryUtil.allocateMemory(0)); @Override public AllocationManager create(BufferAllocator accountingAllocator, long size) { return new AllocationManager(accountingAllocator) { private final long allocatedSize = size; - private final long address = MemoryUtil.UNSAFE.allocateMemory(size); + private final long address = MemoryUtil.allocateMemory(size); @Override public long getSize() { @@ -47,7 +47,7 @@ protected long memoryAddress() { @Override protected void release0() { - MemoryUtil.UNSAFE.freeMemory(address); + MemoryUtil.freeMemory(address); } }; } diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java index d7d7fde00ac63..a5fbc67c48f5c 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java @@ -25,7 +25,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -import java.lang.reflect.Field; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -34,9 +33,9 @@ import org.apache.arrow.memory.rounding.RoundingPolicy; import org.apache.arrow.memory.rounding.SegmentRoundingPolicy; import org.apache.arrow.memory.util.AssertionUtil; +import org.apache.arrow.memory.util.MemoryUtil; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import sun.misc.Unsafe; public class TestBaseAllocator { @@ -405,8 +404,7 @@ private BaseAllocator createAllocatorWithCustomizedAllocationManager() { public AllocationManager create( BufferAllocator accountingAllocator, long requestedSize) { return new AllocationManager(accountingAllocator) { - private final Unsafe unsafe = getUnsafe(); - private final long address = unsafe.allocateMemory(requestedSize); + private final long address = MemoryUtil.allocateMemory(requestedSize); @Override protected long memoryAddress() { @@ -415,29 +413,14 @@ protected long memoryAddress() { @Override protected void release0() { - unsafe.setMemory(address, requestedSize, (byte) 0); - unsafe.freeMemory(address); + MemoryUtil.setMemory(address, requestedSize, (byte) 0); + MemoryUtil.freeMemory(address); } @Override public long getSize() { return requestedSize; } - - private Unsafe getUnsafe() { - Field f = null; - try { - f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - return (Unsafe) f.get(null); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException(e); - } finally { - if (f != null) { - f.setAccessible(false); - } - } - } }; } diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestForeignAllocation.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestForeignAllocation.java index 162bbbcbe939c..b19453df5e109 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestForeignAllocation.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestForeignAllocation.java @@ -139,13 +139,13 @@ private static class UnsafeForeignAllocation extends ForeignAllocation { boolean released = false; public UnsafeForeignAllocation(long bufferSize) { - super(bufferSize, MemoryUtil.UNSAFE.allocateMemory(bufferSize)); + super(bufferSize, MemoryUtil.allocateMemory(bufferSize)); } @Override protected void release0() { if (!released) { - MemoryUtil.UNSAFE.freeMemory(memoryAddress()); + MemoryUtil.freeMemory(memoryAddress()); released = true; } } diff --git a/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/unsafe/UnsafeAllocationManager.java b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/unsafe/UnsafeAllocationManager.java index 31af262a5720e..67d7e0d2af7cb 100644 --- a/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/unsafe/UnsafeAllocationManager.java +++ b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/unsafe/UnsafeAllocationManager.java @@ -26,7 +26,7 @@ public final class UnsafeAllocationManager extends AllocationManager { private static final ArrowBuf EMPTY = - new ArrowBuf(ReferenceManager.NO_OP, null, 0, MemoryUtil.UNSAFE.allocateMemory(0)); + new ArrowBuf(ReferenceManager.NO_OP, null, 0, MemoryUtil.allocateMemory(0)); public static final AllocationManager.Factory FACTORY = new Factory() { @@ -47,7 +47,7 @@ public ArrowBuf empty() { UnsafeAllocationManager(BufferAllocator accountingAllocator, long requestedSize) { super(accountingAllocator); - allocatedAddress = MemoryUtil.UNSAFE.allocateMemory(requestedSize); + allocatedAddress = MemoryUtil.allocateMemory(requestedSize); allocatedSize = requestedSize; } @@ -63,6 +63,6 @@ protected long memoryAddress() { @Override protected void release0() { - MemoryUtil.UNSAFE.freeMemory(allocatedAddress); + MemoryUtil.freeMemory(allocatedAddress); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index 50ddf30bf7e7c..5c74a80fad016 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -888,7 +888,7 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { BitVectorHelper.unsetBit(this.getValidityBuffer(), thisIndex); } else { BitVectorHelper.setBit(this.getValidityBuffer(), thisIndex); - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( from.getDataBuffer().memoryAddress() + (long) fromIndex * typeWidth, this.getDataBuffer().memoryAddress() + (long) thisIndex * typeWidth, typeWidth); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index cb7ef62013de5..0ac56691a6f6c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -247,7 +247,7 @@ public static boolean checkAllBitsEqualTo( int index = 0; while (index + 8 <= fullBytesCount) { - long longValue = MemoryUtil.UNSAFE.getLong(validityBuffer.memoryAddress() + index); + long longValue = MemoryUtil.getLong(validityBuffer.memoryAddress() + index); if (longValue != (long) intToCompare) { return false; } @@ -255,7 +255,7 @@ public static boolean checkAllBitsEqualTo( } if (index + 4 <= fullBytesCount) { - int intValue = MemoryUtil.UNSAFE.getInt(validityBuffer.memoryAddress() + index); + int intValue = MemoryUtil.getInt(validityBuffer.memoryAddress() + index); if (intValue != intToCompare) { return false; } @@ -263,7 +263,7 @@ public static boolean checkAllBitsEqualTo( } while (index < fullBytesCount) { - byte byteValue = MemoryUtil.UNSAFE.getByte(validityBuffer.memoryAddress() + index); + byte byteValue = MemoryUtil.getByte(validityBuffer.memoryAddress() + index); if (byteValue != (byte) intToCompare) { return false; } @@ -272,7 +272,7 @@ public static boolean checkAllBitsEqualTo( // handling with the last bits if (remainder != 0) { - byte byteValue = MemoryUtil.UNSAFE.getByte(validityBuffer.memoryAddress() + sizeInBytes - 1); + byte byteValue = MemoryUtil.getByte(validityBuffer.memoryAddress() + sizeInBytes - 1); byte mask = (byte) ((1 << remainder) - 1); byteValue = (byte) (byteValue & mask); if (checkOneBits) { @@ -386,7 +386,7 @@ public static void concatBits( // copy the first bit set if (input1 != output) { - MemoryUtil.UNSAFE.copyMemory(input1.memoryAddress(), output.memoryAddress(), numBytes1); + MemoryUtil.copyMemory(input1.memoryAddress(), output.memoryAddress(), numBytes1); } if (bitIndex(numBits1) == 0) { @@ -394,8 +394,7 @@ public static void concatBits( // boundary. // For this case, we have a shortcut to copy all bytes from the second set after the byte // boundary. - MemoryUtil.UNSAFE.copyMemory( - input2.memoryAddress(), output.memoryAddress() + numBytes1, numBytes2); + MemoryUtil.copyMemory(input2.memoryAddress(), output.memoryAddress() + numBytes1, numBytes2); return; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java index 8774956522aef..42ad741c85f8b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java @@ -213,13 +213,13 @@ public void setBigEndian(int index, byte[] value) { long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH; if (length == 0) { - MemoryUtil.UNSAFE.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH, (byte) 0); + MemoryUtil.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH, (byte) 0); return; } if (LITTLE_ENDIAN) { // swap bytes to convert BE to LE for (int byteIdx = 0; byteIdx < length; ++byteIdx) { - MemoryUtil.UNSAFE.putByte(outAddress + byteIdx, value[length - 1 - byteIdx]); + MemoryUtil.putByte(outAddress + byteIdx, value[length - 1 - byteIdx]); } if (length == TYPE_WIDTH) { @@ -229,21 +229,17 @@ public void setBigEndian(int index, byte[] value) { if (length < TYPE_WIDTH) { // sign extend final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad); return; } } else { if (length <= TYPE_WIDTH) { // copy data from value to outAddress - MemoryUtil.UNSAFE.copyMemory( - value, - MemoryUtil.BYTE_ARRAY_BASE_OFFSET, - null, - outAddress + Decimal256Vector.TYPE_WIDTH - length, - length); + MemoryUtil.copyToMemory( + value, 0, outAddress + Decimal256Vector.TYPE_WIDTH - length, length); // sign extend final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad); return; } } @@ -282,21 +278,20 @@ public void setSafe(int index, long start, ArrowBuf buffer, int length) { long inAddress = buffer.memoryAddress() + start; long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH; if (LITTLE_ENDIAN) { - MemoryUtil.UNSAFE.copyMemory(inAddress, outAddress, length); + MemoryUtil.copyMemory(inAddress, outAddress, length); // sign extend if (length < TYPE_WIDTH) { - byte msb = MemoryUtil.UNSAFE.getByte(inAddress + length - 1); + byte msb = MemoryUtil.getByte(inAddress + length - 1); final byte pad = (byte) (msb < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad); } } else { - MemoryUtil.UNSAFE.copyMemory( - inAddress, outAddress + Decimal256Vector.TYPE_WIDTH - length, length); + MemoryUtil.copyMemory(inAddress, outAddress + Decimal256Vector.TYPE_WIDTH - length, length); // sign extend if (length < TYPE_WIDTH) { - byte msb = MemoryUtil.UNSAFE.getByte(inAddress); + byte msb = MemoryUtil.getByte(inAddress); final byte pad = (byte) (msb < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad); } } } @@ -323,23 +318,22 @@ public void setBigEndianSafe(int index, long start, ArrowBuf buffer, int length) if (LITTLE_ENDIAN) { // swap bytes to convert BE to LE for (int byteIdx = 0; byteIdx < length; ++byteIdx) { - byte val = MemoryUtil.UNSAFE.getByte((inAddress + length - 1) - byteIdx); - MemoryUtil.UNSAFE.putByte(outAddress + byteIdx, val); + byte val = MemoryUtil.getByte((inAddress + length - 1) - byteIdx); + MemoryUtil.putByte(outAddress + byteIdx, val); } // sign extend if (length < 32) { - byte msb = MemoryUtil.UNSAFE.getByte(inAddress); + byte msb = MemoryUtil.getByte(inAddress); final byte pad = (byte) (msb < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress + length, Decimal256Vector.TYPE_WIDTH - length, pad); } } else { - MemoryUtil.UNSAFE.copyMemory( - inAddress, outAddress + Decimal256Vector.TYPE_WIDTH - length, length); + MemoryUtil.copyMemory(inAddress, outAddress + Decimal256Vector.TYPE_WIDTH - length, length); // sign extend if (length < TYPE_WIDTH) { - byte msb = MemoryUtil.UNSAFE.getByte(inAddress); + byte msb = MemoryUtil.getByte(inAddress); final byte pad = (byte) (msb < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress, Decimal256Vector.TYPE_WIDTH - length, pad); } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index c2f4a14de7cc7..b4c55680b7305 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -212,13 +212,13 @@ public void setBigEndian(int index, byte[] value) { long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH; if (length == 0) { - MemoryUtil.UNSAFE.setMemory(outAddress, DecimalVector.TYPE_WIDTH, (byte) 0); + MemoryUtil.setMemory(outAddress, DecimalVector.TYPE_WIDTH, (byte) 0); return; } if (LITTLE_ENDIAN) { // swap bytes to convert BE to LE for (int byteIdx = 0; byteIdx < length; ++byteIdx) { - MemoryUtil.UNSAFE.putByte(outAddress + byteIdx, value[length - 1 - byteIdx]); + MemoryUtil.putByte(outAddress + byteIdx, value[length - 1 - byteIdx]); } if (length == TYPE_WIDTH) { @@ -228,21 +228,16 @@ public void setBigEndian(int index, byte[] value) { if (length < TYPE_WIDTH) { // sign extend final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad); return; } } else { if (length <= TYPE_WIDTH) { // copy data from value to outAddress - MemoryUtil.UNSAFE.copyMemory( - value, - MemoryUtil.BYTE_ARRAY_BASE_OFFSET, - null, - outAddress + DecimalVector.TYPE_WIDTH - length, - length); + MemoryUtil.copyToMemory(value, 0, outAddress + DecimalVector.TYPE_WIDTH - length, length); // sign extend final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress, DecimalVector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress, DecimalVector.TYPE_WIDTH - length, pad); return; } } @@ -281,21 +276,20 @@ public void setSafe(int index, long start, ArrowBuf buffer, int length) { long inAddress = buffer.memoryAddress() + start; long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH; if (LITTLE_ENDIAN) { - MemoryUtil.UNSAFE.copyMemory(inAddress, outAddress, length); + MemoryUtil.copyMemory(inAddress, outAddress, length); // sign extend if (length < TYPE_WIDTH) { - byte msb = MemoryUtil.UNSAFE.getByte(inAddress + length - 1); + byte msb = MemoryUtil.getByte(inAddress + length - 1); final byte pad = (byte) (msb < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad); } } else { - MemoryUtil.UNSAFE.copyMemory( - inAddress, outAddress + DecimalVector.TYPE_WIDTH - length, length); + MemoryUtil.copyMemory(inAddress, outAddress + DecimalVector.TYPE_WIDTH - length, length); // sign extend if (length < TYPE_WIDTH) { - byte msb = MemoryUtil.UNSAFE.getByte(inAddress); + byte msb = MemoryUtil.getByte(inAddress); final byte pad = (byte) (msb < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress, DecimalVector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress, DecimalVector.TYPE_WIDTH - length, pad); } } } @@ -322,23 +316,22 @@ public void setBigEndianSafe(int index, long start, ArrowBuf buffer, int length) if (LITTLE_ENDIAN) { // swap bytes to convert BE to LE for (int byteIdx = 0; byteIdx < length; ++byteIdx) { - byte val = MemoryUtil.UNSAFE.getByte((inAddress + length - 1) - byteIdx); - MemoryUtil.UNSAFE.putByte(outAddress + byteIdx, val); + byte val = MemoryUtil.getByte((inAddress + length - 1) - byteIdx); + MemoryUtil.putByte(outAddress + byteIdx, val); } // sign extend if (length < TYPE_WIDTH) { - byte msb = MemoryUtil.UNSAFE.getByte(inAddress); + byte msb = MemoryUtil.getByte(inAddress); final byte pad = (byte) (msb < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad); } } else { - MemoryUtil.UNSAFE.copyMemory( - inAddress, outAddress + DecimalVector.TYPE_WIDTH - length, length); + MemoryUtil.copyMemory(inAddress, outAddress + DecimalVector.TYPE_WIDTH - length, length); // sign extend if (length < TYPE_WIDTH) { - byte msb = MemoryUtil.UNSAFE.getByte(inAddress); + byte msb = MemoryUtil.getByte(inAddress); final byte pad = (byte) (msb < 0 ? 0xFF : 0x00); - MemoryUtil.UNSAFE.setMemory(outAddress, DecimalVector.TYPE_WIDTH - length, pad); + MemoryUtil.setMemory(outAddress, DecimalVector.TYPE_WIDTH - length, pad); } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java index dd86b58b267fb..31b79fe53a4a5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java @@ -170,15 +170,15 @@ public static void writeLongToArrowBuf(long value, ArrowBuf bytebuf, int index, final long addressOfValue = bytebuf.memoryAddress() + (long) index * byteWidth; final long padValue = Long.signum(value) == -1 ? -1L : 0L; if (LITTLE_ENDIAN) { - MemoryUtil.UNSAFE.putLong(addressOfValue, value); + MemoryUtil.putLong(addressOfValue, value); for (int i = 1; i <= (byteWidth - 8) / 8; i++) { - MemoryUtil.UNSAFE.putLong(addressOfValue + Long.BYTES * i, padValue); + MemoryUtil.putLong(addressOfValue + Long.BYTES * i, padValue); } } else { for (int i = 0; i < (byteWidth - 8) / 8; i++) { - MemoryUtil.UNSAFE.putLong(addressOfValue + Long.BYTES * i, padValue); + MemoryUtil.putLong(addressOfValue + Long.BYTES * i, padValue); } - MemoryUtil.UNSAFE.putLong(addressOfValue + Long.BYTES * (byteWidth - 8) / 8, value); + MemoryUtil.putLong(addressOfValue + Long.BYTES * (byteWidth - 8) / 8, value); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java b/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java index 4f81cba55f1b3..e703571b374eb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java @@ -93,7 +93,7 @@ public ValueVector visit(BaseFixedWidthVector deltaVector, Void value) { targetVector.getDataBuffer()); } else { - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( deltaVector.getDataBuffer().memoryAddress(), targetVector.getDataBuffer().memoryAddress() + deltaVector.getTypeWidth() * targetVector.getValueCount(), @@ -142,13 +142,13 @@ public ValueVector visit(BaseVariableWidthVector deltaVector, Void value) { targetVector.getValidityBuffer()); // append data buffer - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( deltaVector.getDataBuffer().memoryAddress(), targetVector.getDataBuffer().memoryAddress() + targetDataSize, deltaDataSize); // copy offset buffer - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( deltaVector.getOffsetBuffer().memoryAddress() + BaseVariableWidthVector.OFFSET_WIDTH, targetVector.getOffsetBuffer().memoryAddress() + (targetVector.getValueCount() + 1) * BaseVariableWidthVector.OFFSET_WIDTH, @@ -214,13 +214,13 @@ public ValueVector visit(BaseLargeVariableWidthVector deltaVector, Void value) { targetVector.getValidityBuffer()); // append data buffer - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( deltaVector.getDataBuffer().memoryAddress(), targetVector.getDataBuffer().memoryAddress() + targetDataSize, deltaDataSize); // copy offset buffer - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( deltaVector.getOffsetBuffer().memoryAddress() + BaseLargeVariableWidthVector.OFFSET_WIDTH, targetVector.getOffsetBuffer().memoryAddress() + (targetVector.getValueCount() + 1) * BaseLargeVariableWidthVector.OFFSET_WIDTH, @@ -292,7 +292,7 @@ public ValueVector visit(ListVector deltaVector, Void value) { targetVector.getValidityBuffer()); // append offset buffer - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( deltaVector.getOffsetBuffer().memoryAddress() + ListVector.OFFSET_WIDTH, targetVector.getOffsetBuffer().memoryAddress() + (targetVector.getValueCount() + 1) * ListVector.OFFSET_WIDTH, @@ -362,7 +362,7 @@ public ValueVector visit(LargeListVector deltaVector, Void value) { targetVector.getValidityBuffer()); // append offset buffer - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( deltaVector.getOffsetBuffer().memoryAddress() + ListVector.OFFSET_WIDTH, targetVector.getOffsetBuffer().memoryAddress() + (targetVector.getValueCount() + 1) * LargeListVector.OFFSET_WIDTH, @@ -499,7 +499,7 @@ public ValueVector visit(UnionVector deltaVector, Void value) { } // append type buffers - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( deltaVector.getTypeBufferAddress(), targetUnionVector.getTypeBufferAddress() + targetVector.getValueCount(), deltaVector.getValueCount()); @@ -564,7 +564,7 @@ public ValueVector visit(DenseUnionVector deltaVector, Void value) { } // append type buffers - MemoryUtil.UNSAFE.copyMemory( + MemoryUtil.copyMemory( deltaVector.getTypeBuffer().memoryAddress(), targetDenseUnionVector.getTypeBuffer().memoryAddress() + targetVector.getValueCount(), deltaVector.getValueCount()); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java index f17c065d4e2df..68aa61962ba3f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java @@ -115,34 +115,34 @@ public void testAllBitsSet() { try (RootAllocator allocator = new RootAllocator(bufferLength); ArrowBuf validityBuffer = allocator.buffer(bufferLength)) { - MemoryUtil.UNSAFE.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); + MemoryUtil.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); int bitLength = 1024; assertTrue(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); bitLength = 1028; assertTrue(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); - MemoryUtil.UNSAFE.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); + MemoryUtil.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); bitLength = 1025; BitVectorHelper.unsetBit(validityBuffer, 12); assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); - MemoryUtil.UNSAFE.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); + MemoryUtil.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); bitLength = 1025; BitVectorHelper.unsetBit(validityBuffer, 1024); assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); - MemoryUtil.UNSAFE.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); + MemoryUtil.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); bitLength = 1026; BitVectorHelper.unsetBit(validityBuffer, 1024); assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); - MemoryUtil.UNSAFE.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); + MemoryUtil.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); bitLength = 1027; BitVectorHelper.unsetBit(validityBuffer, 1025); assertFalse(BitVectorHelper.checkAllBitsEqualTo(validityBuffer, bitLength, true)); - MemoryUtil.UNSAFE.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); + MemoryUtil.setMemory(validityBuffer.memoryAddress(), bufferLength, (byte) -1); bitLength = 1031; BitVectorHelper.unsetBit(validityBuffer, 1029); BitVectorHelper.unsetBit(validityBuffer, 1030);