Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GH-38246: [JAVA] added new getTransferPair() function that takes in a Field type for Complex Type Vectors #38261

Merged
merged 9 commits into from
Oct 20, 2023
22 changes: 19 additions & 3 deletions java/vector/src/main/codegen/templates/DenseUnionVector.java
Original file line number Diff line number Diff line change
Expand Up @@ -234,8 +234,8 @@ public synchronized byte registerNewTypeId(Field field) {
typeFields.length + " relative types. Please use union of union instead");
}
byte typeId = nextTypeId;
if (fieldType != null) {
int[] typeIds = ((ArrowType.Union) fieldType.getType()).getTypeIds();
if (this.fieldType != null) {
int[] typeIds = ((ArrowType.Union) this.fieldType.getType()).getTypeIds();
if (typeIds != null) {
int thisTypeId = typeIds[nextTypeId];
if (thisTypeId > Byte.MAX_VALUE) {
Expand Down Expand Up @@ -533,7 +533,7 @@ public Field getField() {
} else {
final UnionMode mode = UnionMode.Dense;
fieldType = new FieldType(this.fieldType.isNullable(), new ArrowType.Union(mode, typeIds),
this.fieldType.getDictionary(), this.fieldType.getMetadata());
this.fieldType.getDictionary(), this.fieldType.getMetadata());
}

return new Field(name, fieldType, childFields);
Expand All @@ -554,6 +554,16 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallB
return new org.apache.arrow.vector.complex.DenseUnionVector.TransferImpl(ref, allocator, callBack);
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator) {
return getTransferPair(field, allocator, null);
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) {
return new org.apache.arrow.vector.complex.DenseUnionVector.TransferImpl(field, allocator, callBack);
}

@Override
public TransferPair makeTransferPair(ValueVector target) {
return new TransferImpl((DenseUnionVector) target);
Expand Down Expand Up @@ -598,6 +608,12 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
createTransferPairs();
}

public TransferImpl(Field field, BufferAllocator allocator, CallBack callBack) {
to = new DenseUnionVector(field.getName(), allocator, null, callBack);
internalStruct.makeTransferPair(to.internalStruct);
createTransferPairs();
}

public TransferImpl(DenseUnionVector to) {
this.to = to;
internalStruct.makeTransferPair(to.internalStruct);
Expand Down
19 changes: 17 additions & 2 deletions java/vector/src/main/codegen/templates/UnionVector.java
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,8 @@ public void initializeChildrenFromFields(List<Field> children) {
int count = 0;
for (Field child: children) {
int typeId = Types.getMinorTypeForArrowType(child.getType()).ordinal();
if (fieldType != null) {
int[] typeIds = ((ArrowType.Union)fieldType.getType()).getTypeIds();
if (this.fieldType != null) {
int[] typeIds = ((ArrowType.Union)this.fieldType.getType()).getTypeIds();
if (typeIds != null) {
typeId = typeIds[count++];
}
Expand Down Expand Up @@ -495,6 +495,16 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallB
return new org.apache.arrow.vector.complex.UnionVector.TransferImpl(ref, allocator, callBack);
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator) {
return getTransferPair(field, allocator, null);
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) {
return new org.apache.arrow.vector.complex.UnionVector.TransferImpl(field, allocator, callBack);
}

@Override
public TransferPair makeTransferPair(ValueVector target) {
return new TransferImpl((UnionVector) target);
Expand Down Expand Up @@ -547,6 +557,11 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
internalStructVectorTransferPair = internalStruct.makeTransferPair(to.internalStruct);
}

public TransferImpl(Field field, BufferAllocator allocator, CallBack callBack) {
to = new UnionVector(field.getName(), allocator, null, callBack);
internalStructVectorTransferPair = internalStruct.makeTransferPair(to.internalStruct);
}

public TransferImpl(UnionVector to) {
this.to = to;
internalStructVectorTransferPair = internalStruct.makeTransferPair(to.internalStruct);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -569,6 +569,18 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallB
return getTransferPair(ref, allocator);
}

/**
* Construct a transfer pair of this vector and another vector of same type.
* @param field The field materialized by this vector.
* @param allocator allocator for the target vector
* @param callBack not used
* @return TransferPair
*/
@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) {
return getTransferPair(field, allocator);
}

/**
* Construct a transfer pair of this vector and another vector of same type.
* @param allocator allocator for the target vector
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -662,6 +662,18 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallB
return getTransferPair(ref, allocator);
}

/**
* Construct a transfer pair of this vector and another vector of same type.
* @param field The field materialized by this vector
* @param allocator allocator for the target vector
* @param callBack not used
* @return TransferPair
*/
@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) {
return getTransferPair(field, allocator);
}

/**
* Construct a transfer pair of this vector and another vector of same type.
* @param allocator allocator for the target vector
Expand All @@ -672,6 +684,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) {
return getTransferPair(getName(), allocator);
}


/**
* Construct a transfer pair of this vector and another vector of same type.
* @param ref name of the target vector
Expand All @@ -680,6 +693,14 @@ public TransferPair getTransferPair(BufferAllocator allocator) {
*/
public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator);

/**
* Construct a transfer pair of this vector and another vector of same type.
* @param field The field materialized by this vector
* @param allocator allocator for the target vector
* @return TransferPair
*/
public abstract TransferPair getTransferPair(Field field, BufferAllocator allocator);

/**
* Transfer this vector'data to another vector. The memory associated
* with this vector is transferred to the allocator of target vector
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -692,6 +692,18 @@ public void validateScalars() {
// No validation by default.
}

/**
* Construct a transfer pair of this vector and another vector of same type.
* @param field The field materialized by this vector.
* @param allocator allocator for the target vector
* @param callBack not used
* @return TransferPair
*/
@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) {
return getTransferPair(field, allocator);
}

/**
* Construct a transfer pair of this vector and another vector of same type.
* @param ref name of the target vector
Expand Down Expand Up @@ -722,6 +734,14 @@ public TransferPair getTransferPair(BufferAllocator allocator) {
*/
public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator);

/**
* Construct a transfer pair of this vector and another vector of same type.
* @param field The field materialized by this vector.
* @param allocator allocator for the target vector
* @return TransferPair
*/
public abstract TransferPair getTransferPair(Field field, BufferAllocator allocator);

/**
* Transfer this vector'data to another vector. The memory associated
* with this vector is transferred to the allocator of target vector
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,16 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallB
return underlyingVector.getTransferPair(ref, allocator, callBack);
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator) {
return underlyingVector.getTransferPair(field, allocator);
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) {
return underlyingVector.getTransferPair(field, allocator, callBack);
}

@Override
public TransferPair makeTransferPair(ValueVector target) {
return underlyingVector.makeTransferPair(target);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,11 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
return new TransferImpl(ref, allocator);
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator) {
return new TransferImpl(field, allocator);
}

/**
* Construct a TransferPair with a desired target vector of the same type.
*
Expand All @@ -271,6 +276,10 @@ public TransferImpl(String ref, BufferAllocator allocator) {
to = new LargeVarBinaryVector(ref, field.getFieldType(), allocator);
}

public TransferImpl(Field field, BufferAllocator allocator) {
to = new LargeVarBinaryVector(field, allocator);
}

public TransferImpl(LargeVarBinaryVector to) {
this.to = to;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,11 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
return new LargeVarCharVector.TransferImpl(ref, allocator);
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator) {
return new LargeVarCharVector.TransferImpl(field, allocator);
}

/**
* Construct a TransferPair with a desired target vector of the same type.
*
Expand All @@ -310,6 +315,10 @@ public TransferImpl(String ref, BufferAllocator allocator) {
to = new LargeVarCharVector(ref, field.getFieldType(), allocator);
}

public TransferImpl(Field field, BufferAllocator allocator) {
to = new LargeVarCharVector(field, allocator);
}

public TransferImpl(LargeVarCharVector to) {
this.to = to;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ public Types.MinorType getMinorType() {

@Override
public TransferPair getTransferPair(BufferAllocator allocator) {
return getTransferPair(null, allocator);
return getTransferPair((String) null, allocator);
}

@Override
Expand Down Expand Up @@ -162,11 +162,21 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
return new TransferImpl();
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator) {
return new TransferImpl();
}

@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
return getTransferPair(ref, allocator);
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack) {
return getTransferPair(field, allocator);
}

@Override
public TransferPair makeTransferPair(ValueVector target) {
return new TransferImpl((NullVector) target);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,48 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
*/
TransferPair getTransferPair(BufferAllocator allocator);

/**
* To transfer quota responsibility.
*
* @param ref the name of the vector
* @param allocator the target allocator
* @return a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
* the same type.
*/
TransferPair getTransferPair(String ref, BufferAllocator allocator);

/**
* To transfer quota responsibility.
*
* @param field the Field object used by the target vector
* @param allocator the target allocator
* @return a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
* the same type.
*/
TransferPair getTransferPair(Field field, BufferAllocator allocator);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, it's unfortunate that this makes existing callsites ambiguous...we should've renamed it in the original PR.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While there's no docstring for the existing function, let's add a docstring for the new ones at least.


/**
* To transfer quota responsibility.
*
* @param ref the name of the vector
* @param allocator the target allocator
* @param callBack A schema change callback.
* @return a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
* the same type.
*/
TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack);

/**
* To transfer quota responsibility.
*
* @param field the Field object used by the target vector
* @param allocator the target allocator
* @param callBack A schema change callback.
* @return a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
* the same type.
*/
TransferPair getTransferPair(Field field, BufferAllocator allocator, CallBack callBack);

/**
* Makes a new transfer pair used to transfer underlying buffers.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,11 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
return new TransferImpl(ref, allocator);
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator) {
return new TransferImpl(field, allocator);
}

/**
* Construct a TransferPair with a desired target vector of the same type.
*
Expand All @@ -272,6 +277,10 @@ public TransferImpl(String ref, BufferAllocator allocator) {
to = new VarBinaryVector(ref, field.getFieldType(), allocator);
}

public TransferImpl(Field field, BufferAllocator allocator) {
to = new VarBinaryVector(field, allocator);
}

public TransferImpl(VarBinaryVector to) {
this.to = to;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,11 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
return new TransferImpl(ref, allocator);
}

@Override
public TransferPair getTransferPair(Field field, BufferAllocator allocator) {
return new TransferImpl(field, allocator);
}

/**
* Construct a TransferPair with a desired target vector of the same type.
*
Expand All @@ -310,6 +315,10 @@ public TransferImpl(String ref, BufferAllocator allocator) {
to = new VarCharVector(ref, field.getFieldType(), allocator);
}

public TransferImpl(Field field, BufferAllocator allocator) {
to = new VarCharVector(field, allocator);
}

public TransferImpl(VarCharVector to) {
this.to = to;
}
Expand Down
Loading
Loading