Skip to content

Commit

Permalink
feat: initial spotless integration flight module
Browse files Browse the repository at this point in the history
  • Loading branch information
vibhatha committed Jun 11, 2024
1 parent 5ebc159 commit 8934eb9
Show file tree
Hide file tree
Showing 16 changed files with 92 additions and 65 deletions.
3 changes: 2 additions & 1 deletion java/c/src/main/java/org/apache/arrow/c/ArrayExporter.java
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ void export(ArrowArray array, FieldVector vector, DictionaryProvider dictionaryP

if (buffers != null) {
data.buffers = new ArrayList<>(buffers.size());
data.buffers_ptrs = allocator.buffer((long) (vector.getExportedCDataBuffers()) * Long.BYTES);
data.buffers_ptrs =
allocator.buffer((long) (vector.getExportedCDataBuffers()) * Long.BYTES);
vector.exportCDataBuffers(data.buffers, data.buffers_ptrs, NULL);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.util.AutoCloseables;
Expand Down Expand Up @@ -235,10 +234,11 @@ private List<ArrowBuf> visitVariableWidthView(ArrowType type) {
final long variadicSizeBufferCapacity = numOfVariadicBuffers * Long.BYTES;
List<ArrowBuf> buffers = new ArrayList<>();

try (ArrowBuf variadicSizeBuffer = importBuffer(type, variadicSizeBufferIndex,
variadicSizeBufferCapacity)) {
try (ArrowBuf variadicSizeBuffer =
importBuffer(type, variadicSizeBufferIndex, variadicSizeBufferCapacity)) {
ArrowBuf maybeValidityBuffer = maybeImportBitmap(type);
try (ArrowBuf view = importFixedBytes(type, viewBufferIndex, BaseVariableWidthViewVector.ELEMENT_SIZE)) {
try (ArrowBuf view =
importFixedBytes(type, viewBufferIndex, BaseVariableWidthViewVector.ELEMENT_SIZE)) {
view.getReferenceManager().retain();
buffers.add(maybeValidityBuffer);
buffers.add(view);
Expand Down Expand Up @@ -402,6 +402,7 @@ public List<ArrowBuf> visit(ArrowType.Duration type) {

@Override
public List<ArrowBuf> visit(ArrowType.ListView type) {
throw new UnsupportedOperationException("Importing buffers for view type: " + type + " not supported");
throw new UnsupportedOperationException(
"Importing buffers for view type: " + type + " not supported");
}
}
26 changes: 20 additions & 6 deletions java/c/src/test/java/org/apache/arrow/c/RoundtripTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -538,27 +538,41 @@ private String generateString(String str, int repetition) {
public void testViewVector() {
// ViewVarCharVector with short strings
try (final ViewVarCharVector vector = new ViewVarCharVector("v1", allocator)) {
setVector(vector, "abc".getBytes(StandardCharsets.UTF_8), "def".getBytes(StandardCharsets.UTF_8), null);
setVector(
vector,
"abc".getBytes(StandardCharsets.UTF_8),
"def".getBytes(StandardCharsets.UTF_8),
null);
assertTrue(roundtrip(vector, ViewVarCharVector.class));
}

// ViewVarCharVector with long strings
try (final ViewVarCharVector vector = new ViewVarCharVector("v2", allocator)) {
setVector(vector, "01234567890123".getBytes(StandardCharsets.UTF_8),
"01234567890123567".getBytes(StandardCharsets.UTF_8), null);
setVector(
vector,
"01234567890123".getBytes(StandardCharsets.UTF_8),
"01234567890123567".getBytes(StandardCharsets.UTF_8),
null);
assertTrue(roundtrip(vector, ViewVarCharVector.class));
}

// ViewVarBinaryVector with short values
try (final ViewVarBinaryVector vector = new ViewVarBinaryVector("v3", allocator)) {
setVector(vector, "abc".getBytes(StandardCharsets.UTF_8), "def".getBytes(StandardCharsets.UTF_8), null);
setVector(
vector,
"abc".getBytes(StandardCharsets.UTF_8),
"def".getBytes(StandardCharsets.UTF_8),
null);
assertTrue(roundtrip(vector, ViewVarBinaryVector.class));
}

// ViewVarBinaryVector with long values
try (final ViewVarBinaryVector vector = new ViewVarBinaryVector("v4", allocator)) {
setVector(vector, "01234567890123".getBytes(StandardCharsets.UTF_8),
"01234567890123567".getBytes(StandardCharsets.UTF_8), null);
setVector(
vector,
"01234567890123".getBytes(StandardCharsets.UTF_8),
"01234567890123567".getBytes(StandardCharsets.UTF_8),
null);
assertTrue(roundtrip(vector, ViewVarBinaryVector.class));
}

Expand Down
23 changes: 16 additions & 7 deletions java/c/src/test/java/org/apache/arrow/c/StreamTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,11 @@ public void roundtripStrings() throws Exception {

@Test
public void roundtripStringViews() throws Exception {
final Schema schema = new Schema(Arrays.asList(Field.nullable("ints", new ArrowType.Int(32, true)),
Field.nullable("string_views", new ArrowType.Utf8View())));
final Schema schema =
new Schema(
Arrays.asList(
Field.nullable("ints", new ArrowType.Int(32, true)),
Field.nullable("string_views", new ArrowType.Utf8View())));
final List<ArrowRecordBatch> batches = new ArrayList<>();
try (final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
final IntVector ints = (IntVector) root.getVector(0);
Expand Down Expand Up @@ -175,8 +178,11 @@ public void roundtripStringViews() throws Exception {

@Test
public void roundtripBinaryViews() throws Exception {
final Schema schema = new Schema(Arrays.asList(Field.nullable("ints", new ArrowType.Int(32, true)),
Field.nullable("binary_views", new ArrowType.BinaryView())));
final Schema schema =
new Schema(
Arrays.asList(
Field.nullable("ints", new ArrowType.Int(32, true)),
Field.nullable("binary_views", new ArrowType.BinaryView())));
final List<ArrowRecordBatch> batches = new ArrayList<>();
try (final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
final IntVector ints = (IntVector) root.getVector(0);
Expand All @@ -189,10 +195,13 @@ public void roundtripBinaryViews() throws Exception {
ints.setSafe(2, 4);
ints.setSafe(3, 8);
strs.setSafe(0, new byte[0]);
strs.setSafe(1, new byte[]{97});
strs.setSafe(1, new byte[] {97});
strs.setSafe(2, new byte[] {98, 99, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48, 98, 99});
strs.setSafe(3, new byte[] {100, 101, 102, 103, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48,
100, 101, 102, 103});
strs.setSafe(
3,
new byte[] {
100, 101, 102, 103, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48, 100, 101, 102, 103
});
root.setRowCount(4);
batches.add(unloader.getRecordBatch());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -530,18 +530,17 @@ public List<ArrowBuf> getFieldBuffers() {

/** Set the reader and writer indexes for the inner buffers. */
/**
* Retrieves the export buffer count for the C Data Interface.
* The exported buffers are the validity and value buffers.
* Retrieves the export buffer count for the C Data Interface. The exported buffers are the
* validity and value buffers.
*
* @return the number of buffers to be exported
*/
@Override
public int getExportedCDataBuffers() {
return 2;
}

/**
* Set the reader and writer indexes for the inner buffers.
*/
/** Set the reader and writer indexes for the inner buffers. */
private void setReaderAndWriterIndex() {
validityBuffer.readerIndex(0);
valueBuffer.readerIndex(0);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -340,10 +340,11 @@ public List<ArrowBuf> getFieldBuffers() {
}

/**
* Retrieves the export buffer count for the C Data Interface.
* The exported buffers are the validity buffer, offset, and value buffers.
* Retrieves the export buffer count for the C Data Interface. The exported buffers are the
* validity buffer, offset, and value buffers.
*
* @return the number of buffers to be exported
*/
*/
@Override
public int getExportedCDataBuffers() {
return 3;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -356,8 +356,9 @@ public List<ArrowBuf> getFieldBuffers() {
}

/**
* Retrieves the export buffer count for the C Data Interface.
* The exported buffers are the validity, offset and value buffers.
* Retrieves the export buffer count for the C Data Interface. The exported buffers are the
* validity, offset and value buffers.
*
* @return the number of buffers to be exported
*/
@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1670,24 +1670,17 @@ public <OUT, IN> OUT accept(VectorVisitor<OUT, IN> visitor, IN value) {
}

/**
* Retrieves the export buffer count for the C Data Interface.
* For Variadic types, an additional buffer is kept to store
* the size of each variadic buffer since that information
* cannot be retrieved in the C Data import.
* When there are dataBuffers, the count is set to 3 + dataBuffers.size().
* Three is formed by validity, view, and variadic size buffer.
* If it is not the case, i.e., the dataBuffer is not present;
* four buffers are required.
* They are view buffer, validity buffer,
* empty data buffer, and variadic size buffer.
* Note that the Java library only allocates data buffers
* when long strings are present.
* In the C Data Interface,
* the binary view import expects at least three buffers.
* The variadic size buffer is merely allocated to determine the
* number of elements per each variadic buffer, and it is not part
* of the imported data.
* Thus, an empty data buffer is allocated to meet this requirement.
* Retrieves the export buffer count for the C Data Interface. For Variadic types, an additional
* buffer is kept to store the size of each variadic buffer since that information cannot be
* retrieved in the C Data import. When there are dataBuffers, the count is set to 3 +
* dataBuffers.size(). Three is formed by validity, view, and variadic size buffer. If it is not
* the case, i.e., the dataBuffer is not present; four buffers are required. They are view buffer,
* validity buffer, empty data buffer, and variadic size buffer. Note that the Java library only
* allocates data buffers when long strings are present. In the C Data Interface, the binary view
* import expects at least three buffers. The variadic size buffer is merely allocated to
* determine the number of elements per each variadic buffer, and it is not part of the imported
* data. Thus, an empty data buffer is allocated to meet this requirement.
*
* @return the number of buffers to be exported
*/
@Override
Expand All @@ -1700,13 +1693,13 @@ public int getExportedCDataBuffers() {
}

/**
* Get the data buffer of the vector.
* Note that an additional buffer is appended to store
* the size of each variadic buffer's size.
* Get the data buffer of the vector. Note that an additional buffer is appended to store the size
* of each variadic buffer's size.
*
* @param buffers list of buffers to be exported
* @param buffersPtr buffer to store the pointers to the exported buffers
* @param nullValue null value
*/
*/
@Override
public void exportCDataBuffers(List<ArrowBuf> buffers, ArrowBuf buffersPtr, long nullValue) {
exportBuffer(validityBuffer, buffers, buffersPtr, nullValue, true);
Expand Down Expand Up @@ -1745,7 +1738,8 @@ public void exportCDataBuffers(List<ArrowBuf> buffers, ArrowBuf buffersPtr, long
for (int i = 0; i < valueCount; i++) {
int length = getValueLength(i);
if (length > 12) {
final int bufIndex = viewBuffer.getInt(((long) i * ELEMENT_SIZE) + LENGTH_WIDTH + PREFIX_WIDTH);
final int bufIndex =
viewBuffer.getInt(((long) i * ELEMENT_SIZE) + LENGTH_WIDTH + PREFIX_WIDTH);
long variadicSizeBufIndex = (long) bufIndex * Long.BYTES;
long currentBufLength = variadicSizeBuffer.getLong(variadicSizeBufIndex);
variadicSizeBuffer.setLong(variadicSizeBufIndex, currentBufLength + length);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,7 @@ public List<ArrowBuf> getFieldBuffers() {

/**
* Retrieves the export buffer count for the C Data Interface.
*
* @return the number of buffers to be exported
*/
@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,9 @@ public interface FieldVector extends ValueVector {

/**
* Retrieves the export buffer count for the C Data Interface.
*
* @return the number of variadic buffers
*/
*/
int getExportedCDataBuffers();

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,8 @@ public List<ArrowBuf> getFieldBuffers() {
}

/**
* Retrieves the export buffer count for the C Data Interface.
* No buffers are exported.
* Retrieves the export buffer count for the C Data Interface. No buffers are exported.
*
* @return the number of buffers to be exported
*/
@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,9 @@ public List<ArrowBuf> getFieldBuffers() {
}

/**
* Retrieves the export buffer count for the C Data Interface.
* The exported buffer is the validity buffer.
* Retrieves the export buffer count for the C Data Interface. The exported buffer is the validity
* buffer.
*
* @return the number of buffers to be exported
*/
public int getExportedCDataBuffers() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -282,8 +282,9 @@ public List<ArrowBuf> getFieldBuffers() {
}

/**
* Retrieves the export buffer count for the C Data Interface.
* The exported buffers are the validity and offset buffers.
* Retrieves the export buffer count for the C Data Interface. The exported buffers are the
* validity and offset buffers.
*
* @return the number of buffers to be exported
*/
@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -239,8 +239,9 @@ public List<ArrowBuf> getFieldBuffers() {
}

/**
* Retrieves the export buffer count for the C Data Interface.
* The exported buffers are the validity and offset buffers.
* Retrieves the export buffer count for the C Data Interface. The exported buffers are the
* validity and offset buffers.
*
* @return the number of buffers to be exported
*/
@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -237,8 +237,9 @@ public List<ArrowBuf> getFieldBuffers() {
}

/**
* Retrieves the export buffer count for the C Data Interface.
* The exported buffers are the validity, offset and size buffers.
* Retrieves the export buffer count for the C Data Interface. The exported buffers are the
* validity, offset and size buffers.
*
* @return the number of buffers to be exported
*/
@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -179,8 +179,9 @@ public List<ArrowBuf> getFieldBuffers() {
}

/**
* Retrieves the export buffer count for the C Data Interface.
* The exported buffer is the validity buffer.
* Retrieves the export buffer count for the C Data Interface. The exported buffer is the validity
* buffer.
*
* @return the number of buffers to be exported
*/
@Override
Expand Down

0 comments on commit 8934eb9

Please sign in to comment.