Skip to content

Commit

Permalink
Fix accountancy in block and vector builders (elastic#100135)
Browse files Browse the repository at this point in the history
  • Loading branch information
ChrisHegarty authored Oct 2, 2023
1 parent 04e478a commit a12315f
Show file tree
Hide file tree
Showing 19 changed files with 193 additions and 229 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -183,26 +183,17 @@ public BooleanBlock build() {
finish();
BooleanBlock block;
if (hasNonNullValue && positionCount == 1 && valueCount == 1) {
block = new ConstantBooleanVector(values[0], 1, blockFactory).asBlock();
block = blockFactory.newConstantBooleanBlockWith(values[0], 1, estimatedBytes);
} else {
if (values.length - valueCount > 1024 || valueCount < (values.length / 2)) {
values = Arrays.copyOf(values, valueCount);
}
if (isDense() && singleValued()) {
block = new BooleanArrayVector(values, positionCount, blockFactory).asBlock();
block = blockFactory.newBooleanArrayVector(values, positionCount, estimatedBytes).asBlock();
} else {
block = new BooleanArrayBlock(values, positionCount, firstValueIndexes, nullsMask, mvOrdering, blockFactory);
block = blockFactory.newBooleanArrayBlock(values, positionCount, firstValueIndexes, nullsMask, mvOrdering, estimatedBytes);
}
}
/*
* Update the breaker with the actual bytes used.
* We pass false below even though we've used the bytes. That's weird,
* but if we break here we will throw away the used memory, letting
* it be deallocated. The exception will bubble up and the builder will
* still technically be open, meaning the calling code should close it
* which will return all used memory to the breaker.
*/
blockFactory.adjustBreaker(block.ramBytesUsed() - estimatedBytes, false);
built();
return block;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,22 +52,13 @@ public BooleanVector build() {
finish();
BooleanVector vector;
if (valueCount == 1) {
vector = new ConstantBooleanVector(values[0], 1, blockFactory);
vector = blockFactory.newConstantBooleanBlockWith(values[0], 1, estimatedBytes).asVector();
} else {
if (values.length - valueCount > 1024 || valueCount < (values.length / 2)) {
values = Arrays.copyOf(values, valueCount);
}
vector = new BooleanArrayVector(values, valueCount, blockFactory);
vector = blockFactory.newBooleanArrayVector(values, valueCount, estimatedBytes);
}
/*
* Update the breaker with the actual bytes used.
* We pass false below even though we've used the bytes. That's weird,
* but if we break here we will throw away the used memory, letting
* it be deallocated. The exception will bubble up and the builder will
* still technically be open, meaning the calling code should close it
* which will return all used memory to the breaker.
*/
blockFactory.adjustBreaker(vector.ramBytesUsed() - estimatedBytes, false);
built();
return vector;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,16 @@
final class BooleanVectorFixedBuilder implements BooleanVector.FixedBuilder {
private final BlockFactory blockFactory;
private final boolean[] values;
private final long preAdjustedBytes;
/**
* The next value to write into. {@code -1} means the vector has already
* been built.
*/
private int nextIndex;

BooleanVectorFixedBuilder(int size, BlockFactory blockFactory) {
blockFactory.adjustBreaker(ramBytesUsed(size), false);
preAdjustedBytes = ramBytesUsed(size);
blockFactory.adjustBreaker(preAdjustedBytes, false);
this.blockFactory = blockFactory;
this.values = new boolean[size];
}
Expand Down Expand Up @@ -54,16 +56,16 @@ public BooleanVector build() {
}
nextIndex = -1;
if (values.length == 1) {
return new ConstantBooleanVector(values[0], 1, blockFactory);
return blockFactory.newConstantBooleanBlockWith(values[0], 1, preAdjustedBytes).asVector();
}
return new BooleanArrayVector(values, values.length, blockFactory);
return blockFactory.newBooleanArrayVector(values, values.length, preAdjustedBytes);
}

@Override
public void close() {
if (nextIndex >= 0) {
// If nextIndex < 0 we've already built the vector
blockFactory.adjustBreaker(-ramBytesUsed(values.length), false);
blockFactory.adjustBreaker(-preAdjustedBytes, false);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -183,26 +183,17 @@ public DoubleBlock build() {
finish();
DoubleBlock block;
if (hasNonNullValue && positionCount == 1 && valueCount == 1) {
block = new ConstantDoubleVector(values[0], 1, blockFactory).asBlock();
block = blockFactory.newConstantDoubleBlockWith(values[0], 1, estimatedBytes);
} else {
if (values.length - valueCount > 1024 || valueCount < (values.length / 2)) {
values = Arrays.copyOf(values, valueCount);
}
if (isDense() && singleValued()) {
block = new DoubleArrayVector(values, positionCount, blockFactory).asBlock();
block = blockFactory.newDoubleArrayVector(values, positionCount, estimatedBytes).asBlock();
} else {
block = new DoubleArrayBlock(values, positionCount, firstValueIndexes, nullsMask, mvOrdering, blockFactory);
block = blockFactory.newDoubleArrayBlock(values, positionCount, firstValueIndexes, nullsMask, mvOrdering, estimatedBytes);
}
}
/*
* Update the breaker with the actual bytes used.
* We pass false below even though we've used the bytes. That's weird,
* but if we break here we will throw away the used memory, letting
* it be deallocated. The exception will bubble up and the builder will
* still technically be open, meaning the calling code should close it
* which will return all used memory to the breaker.
*/
blockFactory.adjustBreaker(block.ramBytesUsed() - estimatedBytes, false);
built();
return block;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,22 +52,13 @@ public DoubleVector build() {
finish();
DoubleVector vector;
if (valueCount == 1) {
vector = new ConstantDoubleVector(values[0], 1, blockFactory);
vector = blockFactory.newConstantDoubleBlockWith(values[0], 1, estimatedBytes).asVector();
} else {
if (values.length - valueCount > 1024 || valueCount < (values.length / 2)) {
values = Arrays.copyOf(values, valueCount);
}
vector = new DoubleArrayVector(values, valueCount, blockFactory);
vector = blockFactory.newDoubleArrayVector(values, valueCount, estimatedBytes);
}
/*
* Update the breaker with the actual bytes used.
* We pass false below even though we've used the bytes. That's weird,
* but if we break here we will throw away the used memory, letting
* it be deallocated. The exception will bubble up and the builder will
* still technically be open, meaning the calling code should close it
* which will return all used memory to the breaker.
*/
blockFactory.adjustBreaker(vector.ramBytesUsed() - estimatedBytes, false);
built();
return vector;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,16 @@
final class DoubleVectorFixedBuilder implements DoubleVector.FixedBuilder {
private final BlockFactory blockFactory;
private final double[] values;
private final long preAdjustedBytes;
/**
* The next value to write into. {@code -1} means the vector has already
* been built.
*/
private int nextIndex;

DoubleVectorFixedBuilder(int size, BlockFactory blockFactory) {
blockFactory.adjustBreaker(ramBytesUsed(size), false);
preAdjustedBytes = ramBytesUsed(size);
blockFactory.adjustBreaker(preAdjustedBytes, false);
this.blockFactory = blockFactory;
this.values = new double[size];
}
Expand Down Expand Up @@ -54,16 +56,16 @@ public DoubleVector build() {
}
nextIndex = -1;
if (values.length == 1) {
return new ConstantDoubleVector(values[0], 1, blockFactory);
return blockFactory.newConstantDoubleBlockWith(values[0], 1, preAdjustedBytes).asVector();
}
return new DoubleArrayVector(values, values.length, blockFactory);
return blockFactory.newDoubleArrayVector(values, values.length, preAdjustedBytes);
}

@Override
public void close() {
if (nextIndex >= 0) {
// If nextIndex < 0 we've already built the vector
blockFactory.adjustBreaker(-ramBytesUsed(values.length), false);
blockFactory.adjustBreaker(-preAdjustedBytes, false);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -183,26 +183,17 @@ public IntBlock build() {
finish();
IntBlock block;
if (hasNonNullValue && positionCount == 1 && valueCount == 1) {
block = new ConstantIntVector(values[0], 1, blockFactory).asBlock();
block = blockFactory.newConstantIntBlockWith(values[0], 1, estimatedBytes);
} else {
if (values.length - valueCount > 1024 || valueCount < (values.length / 2)) {
values = Arrays.copyOf(values, valueCount);
}
if (isDense() && singleValued()) {
block = new IntArrayVector(values, positionCount, blockFactory).asBlock();
block = blockFactory.newIntArrayVector(values, positionCount, estimatedBytes).asBlock();
} else {
block = new IntArrayBlock(values, positionCount, firstValueIndexes, nullsMask, mvOrdering, blockFactory);
block = blockFactory.newIntArrayBlock(values, positionCount, firstValueIndexes, nullsMask, mvOrdering, estimatedBytes);
}
}
/*
* Update the breaker with the actual bytes used.
* We pass false below even though we've used the bytes. That's weird,
* but if we break here we will throw away the used memory, letting
* it be deallocated. The exception will bubble up and the builder will
* still technically be open, meaning the calling code should close it
* which will return all used memory to the breaker.
*/
blockFactory.adjustBreaker(block.ramBytesUsed() - estimatedBytes, false);
built();
return block;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,22 +52,13 @@ public IntVector build() {
finish();
IntVector vector;
if (valueCount == 1) {
vector = new ConstantIntVector(values[0], 1, blockFactory);
vector = blockFactory.newConstantIntBlockWith(values[0], 1, estimatedBytes).asVector();
} else {
if (values.length - valueCount > 1024 || valueCount < (values.length / 2)) {
values = Arrays.copyOf(values, valueCount);
}
vector = new IntArrayVector(values, valueCount, blockFactory);
vector = blockFactory.newIntArrayVector(values, valueCount, estimatedBytes);
}
/*
* Update the breaker with the actual bytes used.
* We pass false below even though we've used the bytes. That's weird,
* but if we break here we will throw away the used memory, letting
* it be deallocated. The exception will bubble up and the builder will
* still technically be open, meaning the calling code should close it
* which will return all used memory to the breaker.
*/
blockFactory.adjustBreaker(vector.ramBytesUsed() - estimatedBytes, false);
built();
return vector;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,16 @@
final class IntVectorFixedBuilder implements IntVector.FixedBuilder {
private final BlockFactory blockFactory;
private final int[] values;
private final long preAdjustedBytes;
/**
* The next value to write into. {@code -1} means the vector has already
* been built.
*/
private int nextIndex;

IntVectorFixedBuilder(int size, BlockFactory blockFactory) {
blockFactory.adjustBreaker(ramBytesUsed(size), false);
preAdjustedBytes = ramBytesUsed(size);
blockFactory.adjustBreaker(preAdjustedBytes, false);
this.blockFactory = blockFactory;
this.values = new int[size];
}
Expand Down Expand Up @@ -54,16 +56,16 @@ public IntVector build() {
}
nextIndex = -1;
if (values.length == 1) {
return new ConstantIntVector(values[0], 1, blockFactory);
return blockFactory.newConstantIntBlockWith(values[0], 1, preAdjustedBytes).asVector();
}
return new IntArrayVector(values, values.length, blockFactory);
return blockFactory.newIntArrayVector(values, values.length, preAdjustedBytes);
}

@Override
public void close() {
if (nextIndex >= 0) {
// If nextIndex < 0 we've already built the vector
blockFactory.adjustBreaker(-ramBytesUsed(values.length), false);
blockFactory.adjustBreaker(-preAdjustedBytes, false);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -183,26 +183,17 @@ public LongBlock build() {
finish();
LongBlock block;
if (hasNonNullValue && positionCount == 1 && valueCount == 1) {
block = new ConstantLongVector(values[0], 1, blockFactory).asBlock();
block = blockFactory.newConstantLongBlockWith(values[0], 1, estimatedBytes);
} else {
if (values.length - valueCount > 1024 || valueCount < (values.length / 2)) {
values = Arrays.copyOf(values, valueCount);
}
if (isDense() && singleValued()) {
block = new LongArrayVector(values, positionCount, blockFactory).asBlock();
block = blockFactory.newLongArrayVector(values, positionCount, estimatedBytes).asBlock();
} else {
block = new LongArrayBlock(values, positionCount, firstValueIndexes, nullsMask, mvOrdering, blockFactory);
block = blockFactory.newLongArrayBlock(values, positionCount, firstValueIndexes, nullsMask, mvOrdering, estimatedBytes);
}
}
/*
* Update the breaker with the actual bytes used.
* We pass false below even though we've used the bytes. That's weird,
* but if we break here we will throw away the used memory, letting
* it be deallocated. The exception will bubble up and the builder will
* still technically be open, meaning the calling code should close it
* which will return all used memory to the breaker.
*/
blockFactory.adjustBreaker(block.ramBytesUsed() - estimatedBytes, false);
built();
return block;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,22 +52,13 @@ public LongVector build() {
finish();
LongVector vector;
if (valueCount == 1) {
vector = new ConstantLongVector(values[0], 1, blockFactory);
vector = blockFactory.newConstantLongBlockWith(values[0], 1, estimatedBytes).asVector();
} else {
if (values.length - valueCount > 1024 || valueCount < (values.length / 2)) {
values = Arrays.copyOf(values, valueCount);
}
vector = new LongArrayVector(values, valueCount, blockFactory);
vector = blockFactory.newLongArrayVector(values, valueCount, estimatedBytes);
}
/*
* Update the breaker with the actual bytes used.
* We pass false below even though we've used the bytes. That's weird,
* but if we break here we will throw away the used memory, letting
* it be deallocated. The exception will bubble up and the builder will
* still technically be open, meaning the calling code should close it
* which will return all used memory to the breaker.
*/
blockFactory.adjustBreaker(vector.ramBytesUsed() - estimatedBytes, false);
built();
return vector;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,16 @@
final class LongVectorFixedBuilder implements LongVector.FixedBuilder {
private final BlockFactory blockFactory;
private final long[] values;
private final long preAdjustedBytes;
/**
* The next value to write into. {@code -1} means the vector has already
* been built.
*/
private int nextIndex;

LongVectorFixedBuilder(int size, BlockFactory blockFactory) {
blockFactory.adjustBreaker(ramBytesUsed(size), false);
preAdjustedBytes = ramBytesUsed(size);
blockFactory.adjustBreaker(preAdjustedBytes, false);
this.blockFactory = blockFactory;
this.values = new long[size];
}
Expand Down Expand Up @@ -54,16 +56,16 @@ public LongVector build() {
}
nextIndex = -1;
if (values.length == 1) {
return new ConstantLongVector(values[0], 1, blockFactory);
return blockFactory.newConstantLongBlockWith(values[0], 1, preAdjustedBytes).asVector();
}
return new LongArrayVector(values, values.length, blockFactory);
return blockFactory.newLongArrayVector(values, values.length, preAdjustedBytes);
}

@Override
public void close() {
if (nextIndex >= 0) {
// If nextIndex < 0 we've already built the vector
blockFactory.adjustBreaker(-ramBytesUsed(values.length), false);
blockFactory.adjustBreaker(-preAdjustedBytes, false);
}
}
}
Loading

0 comments on commit a12315f

Please sign in to comment.