diff --git a/docs/changelog/103681.yaml b/docs/changelog/103681.yaml new file mode 100644 index 0000000000000..bba73c8e3a7d4 --- /dev/null +++ b/docs/changelog/103681.yaml @@ -0,0 +1,6 @@ +pr: 103681 +summary: "ESQL: Expand shallow copy with vecs" +area: ES|QL +type: enhancement +issues: + - 100528 diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java index f9b8358faee6b..d7d0856963019 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java @@ -29,9 +29,27 @@ final class BooleanArrayBlock extends AbstractArrayBlock implements BooleanBlock BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new BooleanArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private BooleanArrayBlock( + BooleanArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new BooleanArrayVector(values, values.length, blockFactory); + this.vector = vector; } @Override @@ -46,6 +64,7 @@ public boolean getBoolean(int valueIndex) { @Override public BooleanBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newBooleanBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -79,21 +98,28 @@ public BooleanBlock expand() { incRef(); return this; } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newBooleanBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendBoolean(getBoolean(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + + BooleanArrayBlock expanded = new BooleanArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java index 17ed741bd59da..d75e988fe3a84 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java @@ -30,9 +30,27 @@ public BooleanBigArrayBlock( BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new BooleanBigArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private BooleanBigArrayBlock( + BooleanBigArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new BooleanBigArrayVector(values, (int) values.size(), blockFactory); + this.vector = vector; } @Override @@ -47,6 +65,7 @@ public boolean getBoolean(int valueIndex) { @Override public BooleanBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newBooleanBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -80,21 +99,28 @@ public BooleanBlock expand() { incRef(); return this; } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newBooleanBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendBoolean(getBoolean(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + + BooleanBigArrayBlock expanded = new BooleanBigArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java index e2598d3d86b8f..b236949e92018 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java @@ -32,9 +32,27 @@ final class BytesRefArrayBlock extends AbstractArrayBlock implements BytesRefBlo BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new BytesRefArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private BytesRefArrayBlock( + BytesRefArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new BytesRefArrayVector(values, (int) values.size(), blockFactory); + this.vector = vector; } @Override @@ -49,6 +67,7 @@ public BytesRef getBytesRef(int valueIndex, BytesRef dest) { @Override public BytesRefBlock filter(int... positions) { + // TODO use reference counting to share the vector final BytesRef scratch = new BytesRef(); try (var builder = blockFactory().newBytesRefBlockBuilder(positions.length)) { for (int pos : positions) { @@ -83,22 +102,28 @@ public BytesRefBlock expand() { incRef(); return this; } - // TODO use reference counting to share the vector - final BytesRef scratch = new BytesRef(); - try (var builder = blockFactory().newBytesRefBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendBytesRef(getBytesRef(i, scratch)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + + BytesRefArrayBlock expanded = new BytesRefArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java index 96e96ac459a50..6089679904e48 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java @@ -29,9 +29,27 @@ final class DoubleArrayBlock extends AbstractArrayBlock implements DoubleBlock { BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new DoubleArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private DoubleArrayBlock( + DoubleArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new DoubleArrayVector(values, values.length, blockFactory); + this.vector = vector; } @Override @@ -46,6 +64,7 @@ public double getDouble(int valueIndex) { @Override public DoubleBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newDoubleBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -79,21 +98,28 @@ public DoubleBlock expand() { incRef(); return this; } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newDoubleBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendDouble(getDouble(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + + DoubleArrayBlock expanded = new DoubleArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java index 5b1dcbfc9d728..adc09d5755b53 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java @@ -30,9 +30,27 @@ public DoubleBigArrayBlock( BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new DoubleBigArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private DoubleBigArrayBlock( + DoubleBigArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new DoubleBigArrayVector(values, (int) values.size(), blockFactory); + this.vector = vector; } @Override @@ -47,6 +65,7 @@ public double getDouble(int valueIndex) { @Override public DoubleBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newDoubleBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -80,21 +99,28 @@ public DoubleBlock expand() { incRef(); return this; } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newDoubleBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendDouble(getDouble(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + + DoubleBigArrayBlock expanded = new DoubleBigArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java index e8f10ced11adc..1e5ae7a3de448 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java @@ -29,9 +29,27 @@ final class IntArrayBlock extends AbstractArrayBlock implements IntBlock { BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new IntArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private IntArrayBlock( + IntArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new IntArrayVector(values, values.length, blockFactory); + this.vector = vector; } @Override @@ -46,6 +64,7 @@ public int getInt(int valueIndex) { @Override public IntBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newIntBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -79,21 +98,28 @@ public IntBlock expand() { incRef(); return this; } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newIntBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendInt(getInt(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + + IntArrayBlock expanded = new IntArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java index ad6033fb452a0..068f550a56d5b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java @@ -30,9 +30,27 @@ public IntBigArrayBlock( BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new IntBigArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private IntBigArrayBlock( + IntBigArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new IntBigArrayVector(values, (int) values.size(), blockFactory); + this.vector = vector; } @Override @@ -47,6 +65,7 @@ public int getInt(int valueIndex) { @Override public IntBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newIntBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -80,21 +99,28 @@ public IntBlock expand() { incRef(); return this; } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newIntBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendInt(getInt(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + + IntBigArrayBlock expanded = new IntBigArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java index 792f9b267e748..0d8464a95ed5b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java @@ -29,9 +29,27 @@ final class LongArrayBlock extends AbstractArrayBlock implements LongBlock { BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new LongArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private LongArrayBlock( + LongArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new LongArrayVector(values, values.length, blockFactory); + this.vector = vector; } @Override @@ -46,6 +64,7 @@ public long getLong(int valueIndex) { @Override public LongBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newLongBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -79,21 +98,28 @@ public LongBlock expand() { incRef(); return this; } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newLongBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendLong(getLong(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + + LongArrayBlock expanded = new LongArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java index dc19a4038a9e9..eb1a353352ac9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java @@ -30,9 +30,27 @@ public LongBigArrayBlock( BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new LongBigArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private LongBigArrayBlock( + LongBigArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new LongBigArrayVector(values, (int) values.size(), blockFactory); + this.vector = vector; } @Override @@ -47,6 +65,7 @@ public long getLong(int valueIndex) { @Override public LongBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newLongBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -80,21 +99,28 @@ public LongBlock expand() { incRef(); return this; } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newLongBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendLong(getLong(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + + LongBigArrayBlock expanded = new LongBigArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java index bdc4dbef15bd2..d1f1bac940714 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java @@ -23,6 +23,10 @@ public static long sizeOf(@Nullable int[] arr) { /** Returns the size in bytes used by the bitset. Otherwise, returns 0 if null. Not exact, but good enough */ public static long sizeOfBitSet(@Nullable BitSet bitset) { - return bitset == null ? 0 : BITSET_BASE_RAM_USAGE + (bitset.size() / Byte.SIZE); + return bitset == null ? 0 : sizeOfBitSet(bitset.size()); + } + + public static long sizeOfBitSet(long size) { + return BITSET_BASE_RAM_USAGE + (size / Byte.SIZE); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st index 01a6d70d63795..245089ff2a83e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st @@ -40,13 +40,27 @@ final class $Type$ArrayBlock extends AbstractArrayBlock implements $Type$Block { BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new $Type$ArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private $Type$ArrayBlock( + $Type$ArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - $if(BytesRef)$ - this.vector = new BytesRefArrayVector(values, (int) values.size(), blockFactory); - $else$ - this.vector = new $Type$ArrayVector(values, values.length, blockFactory); - $endif$ + this.vector = vector; } @Override @@ -66,6 +80,7 @@ $endif$ @Override public $Type$Block filter(int... positions) { + // TODO use reference counting to share the vector $if(BytesRef)$ final BytesRef scratch = new BytesRef(); $endif$ @@ -102,28 +117,28 @@ $endif$ incRef(); return this; } - // TODO use reference counting to share the vector -$if(BytesRef)$ - final BytesRef scratch = new BytesRef(); -$endif$ - try (var builder = blockFactory().new$Type$BlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { -$if(BytesRef)$ - builder.append$Type$(get$Type$(i, scratch)); -$else$ - builder.append$Type$(get$Type$(i)); -$endif$ - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + + $Type$ArrayBlock expanded = new $Type$ArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st index 989f119bca062..a5f5001802cb7 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st @@ -30,9 +30,27 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new $Type$BigArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private $Type$BigArrayBlock( + $Type$BigArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new $Type$BigArrayVector(values, (int) values.size(), blockFactory); + this.vector = vector; } @Override @@ -47,6 +65,7 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty @Override public $Type$Block filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().new$Type$BlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -80,21 +99,28 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty incRef(); return this; } - // TODO use reference counting to share the vector - try (var builder = blockFactory().new$Type$BlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.append$Type$(get$Type$(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + + $Type$BigArrayBlock expanded = new $Type$BigArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java index 4cde556120465..0547dabd65f21 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java @@ -60,25 +60,25 @@ public void testEmpty() { } void testEmpty(BlockFactory bf) { - assertZeroPositionsAndRelease(bf.newIntArrayBlock(new int[] {}, 0, new int[] {}, new BitSet(), randomOrdering())); + assertZeroPositionsAndRelease(bf.newIntArrayBlock(new int[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering())); assertZeroPositionsAndRelease(bf.newIntBlockBuilder(0).build()); assertZeroPositionsAndRelease(bf.newIntArrayVector(new int[] {}, 0)); assertZeroPositionsAndRelease(bf.newIntVectorBuilder(0).build()); - assertZeroPositionsAndRelease(bf.newLongArrayBlock(new long[] {}, 0, new int[] {}, new BitSet(), randomOrdering())); + assertZeroPositionsAndRelease(bf.newLongArrayBlock(new long[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering())); assertZeroPositionsAndRelease(bf.newLongBlockBuilder(0).build()); assertZeroPositionsAndRelease(bf.newLongArrayVector(new long[] {}, 0)); assertZeroPositionsAndRelease(bf.newLongVectorBuilder(0).build()); - assertZeroPositionsAndRelease(bf.newDoubleArrayBlock(new double[] {}, 0, new int[] {}, new BitSet(), randomOrdering())); + assertZeroPositionsAndRelease(bf.newDoubleArrayBlock(new double[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering())); assertZeroPositionsAndRelease(bf.newDoubleBlockBuilder(0).build()); assertZeroPositionsAndRelease(bf.newDoubleArrayVector(new double[] {}, 0)); assertZeroPositionsAndRelease(bf.newDoubleVectorBuilder(0).build()); assertZeroPositionsAndRelease( - bf.newBytesRefArrayBlock(new BytesRefArray(0, bf.bigArrays()), 0, new int[] {}, new BitSet(), randomOrdering()) + bf.newBytesRefArrayBlock(new BytesRefArray(0, bf.bigArrays()), 0, new int[] { 0 }, new BitSet(), randomOrdering()) ); assertZeroPositionsAndRelease(bf.newBytesRefBlockBuilder(0).build()); assertZeroPositionsAndRelease(bf.newBytesRefArrayVector(new BytesRefArray(0, bf.bigArrays()), 0)); assertZeroPositionsAndRelease(bf.newBytesRefVectorBuilder(0).build()); - assertZeroPositionsAndRelease(bf.newBooleanArrayBlock(new boolean[] {}, 0, new int[] {}, new BitSet(), randomOrdering())); + assertZeroPositionsAndRelease(bf.newBooleanArrayBlock(new boolean[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering())); assertZeroPositionsAndRelease(bf.newBooleanBlockBuilder(0).build()); assertZeroPositionsAndRelease(bf.newBooleanArrayVector(new boolean[] {}, 0)); assertZeroPositionsAndRelease(bf.newBooleanVectorBuilder(0).build()); @@ -1240,13 +1240,14 @@ private Vector randomConstantVector() { private Block randomArrayBlock() { int positionCount = randomIntBetween(0, 100); int arrayType = randomIntBetween(0, 4); + int[] firstValueIndexes = IntStream.range(0, positionCount + 1).toArray(); return switch (arrayType) { case 0 -> { boolean[] values = new boolean[positionCount]; Arrays.fill(values, randomBoolean()); - yield blockFactory.newBooleanArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + yield blockFactory.newBooleanArrayBlock(values, positionCount, firstValueIndexes, new BitSet(), randomOrdering()); } case 1 -> { BytesRefArray values = new BytesRefArray(positionCount, BigArrays.NON_RECYCLING_INSTANCE); @@ -1254,25 +1255,25 @@ private Block randomArrayBlock() { values.append(new BytesRef(randomByteArrayOfLength(between(1, 20)))); } - yield blockFactory.newBytesRefArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + yield blockFactory.newBytesRefArrayBlock(values, positionCount, firstValueIndexes, new BitSet(), randomOrdering()); } case 2 -> { double[] values = new double[positionCount]; Arrays.fill(values, 1.0); - yield blockFactory.newDoubleArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + yield blockFactory.newDoubleArrayBlock(values, positionCount, firstValueIndexes, new BitSet(), randomOrdering()); } case 3 -> { int[] values = new int[positionCount]; Arrays.fill(values, 1); - yield blockFactory.newIntArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + yield blockFactory.newIntArrayBlock(values, positionCount, firstValueIndexes, new BitSet(), randomOrdering()); } default -> { long[] values = new long[positionCount]; Arrays.fill(values, 1L); - yield blockFactory.newLongArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + yield blockFactory.newLongArrayBlock(values, positionCount, firstValueIndexes, new BitSet(), randomOrdering()); } }; } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java index b5155f3199c1c..d34e639c32f0d 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java @@ -134,19 +134,22 @@ public void testBytesRefVector() { // Array Blocks public void testBooleanBlock() { BlockFactory blockFactory = blockFactory(); - Block empty = new BooleanArrayBlock(new boolean[] {}, 0, new int[0], null, Block.MvOrdering.UNORDERED, blockFactory); + Block empty = new BooleanArrayBlock(new boolean[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory); long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR); assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); Block emptyPlusOne = new BooleanArrayBlock( new boolean[] { randomBoolean() }, 1, - new int[] { 0 }, + new int[] { 0, 1 }, null, Block.MvOrdering.UNORDERED, blockFactory ); - assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + 1) + alignObjectSize(Integer.BYTES))); + assertThat( + emptyPlusOne.ramBytesUsed(), + is(empty.ramBytesUsed() + ramBytesDiffForBooleanArrays(1, 0) + ramBytesDiffForIntArrays(2, 1)) + ); boolean[] randomData = new boolean[randomIntBetween(2, 1024)]; int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); @@ -158,7 +161,10 @@ public void testBooleanBlock() { Block.MvOrdering.UNORDERED, blockFactory ); - long expected = empty.ramBytesUsed() + ramBytesForBooleanArray(randomData) + ramBytesForIntArray(valueIndices); + long expected = empty.ramBytesUsed() + ramBytesDiffForBooleanArrays(randomData.length, 0) + ramBytesDiffForIntArrays( + valueIndices.length, + 1 + ); assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); @@ -181,19 +187,19 @@ public void testBooleanBlockWithNullFirstValues() { public void testIntBlock() { BlockFactory blockFactory = blockFactory(); - Block empty = new IntArrayBlock(new int[] {}, 0, new int[] {}, null, Block.MvOrdering.UNORDERED, blockFactory); + Block empty = new IntArrayBlock(new int[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory); long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR); assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); Block emptyPlusOne = new IntArrayBlock( new int[] { randomInt() }, 1, - new int[] { 0 }, + new int[] { 0, 1 }, null, Block.MvOrdering.UNORDERED, blockFactory ); - assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + alignObjectSize(Integer.BYTES) + alignObjectSize(Integer.BYTES))); + assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + ramBytesDiffForIntArrays(1, 0) + ramBytesDiffForIntArrays(2, 1))); int[] randomData = new int[randomIntBetween(2, 1024)]; int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); @@ -205,7 +211,10 @@ public void testIntBlock() { Block.MvOrdering.UNORDERED, blockFactory ); - long expected = empty.ramBytesUsed() + ramBytesForIntArray(randomData) + ramBytesForIntArray(valueIndices); + long expected = empty.ramBytesUsed() + ramBytesDiffForIntArrays(randomData.length, 0) + ramBytesDiffForIntArrays( + valueIndices.length, + 1 + ); assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); @@ -222,19 +231,22 @@ public void testIntBlockWithNullFirstValues() { public void testLongBlock() { BlockFactory blockFactory = blockFactory(); - Block empty = new LongArrayBlock(new long[] {}, 0, new int[0], null, Block.MvOrdering.UNORDERED, blockFactory); + Block empty = new LongArrayBlock(new long[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory); long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR); assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); Block emptyPlusOne = new LongArrayBlock( new long[] { randomInt() }, 1, - new int[] { 0 }, + new int[] { 0, 1 }, null, Block.MvOrdering.UNORDERED, blockFactory ); - assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Long.BYTES) + alignObjectSize(Integer.BYTES))); + assertThat( + emptyPlusOne.ramBytesUsed(), + is(empty.ramBytesUsed() + ramBytesDiffForLongArrays(1, 0) + ramBytesDiffForIntArrays(2, 1)) + ); long[] randomData = new long[randomIntBetween(2, 1024)]; int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); @@ -246,7 +258,10 @@ public void testLongBlock() { Block.MvOrdering.UNORDERED, blockFactory ); - long expected = empty.ramBytesUsed() + ramBytesForLongArray(randomData) + ramBytesForIntArray(valueIndices); + long expected = empty.ramBytesUsed() + ramBytesDiffForLongArrays(randomData.length, 0) + ramBytesDiffForIntArrays( + valueIndices.length, + 1 + ); assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); @@ -269,19 +284,22 @@ public void testLongBlockWithNullFirstValues() { public void testDoubleBlock() { BlockFactory blockFactory = blockFactory(); - Block empty = new DoubleArrayBlock(new double[] {}, 0, new int[0], null, Block.MvOrdering.UNORDERED, blockFactory); + Block empty = new DoubleArrayBlock(new double[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory); long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR); assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); Block emptyPlusOne = new DoubleArrayBlock( new double[] { randomInt() }, 1, - new int[] { 0 }, + new int[] { 0, 1 }, null, Block.MvOrdering.UNORDERED, blockFactory ); - assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Double.BYTES) + alignObjectSize(Integer.BYTES))); + assertThat( + emptyPlusOne.ramBytesUsed(), + is(empty.ramBytesUsed() + ramBytesDiffForDoubleArrays(1, 0) + ramBytesDiffForIntArrays(2, 1)) + ); double[] randomData = new double[randomIntBetween(2, 1024)]; int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); @@ -293,7 +311,10 @@ public void testDoubleBlock() { Block.MvOrdering.UNORDERED, blockFactory ); - long expected = empty.ramBytesUsed() + ramBytesForDoubleArray(randomData) + ramBytesForIntArray(valueIndices); + long expected = empty.ramBytesUsed() + ramBytesDiffForDoubleArrays(randomData.length, 0) + ramBytesDiffForIntArrays( + valueIndices.length, + 1 + ); assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); @@ -344,19 +365,35 @@ public long accumulateObject(Object o, long shallowSize, Map fiel } } - static long ramBytesForBooleanArray(boolean[] arr) { - return alignObjectSize((long) Byte.BYTES * arr.length); + static long ramBytesDiffForBooleanArrays(int length1, int lenght2) { + return ramBytesForBooleanArray(length1) - ramBytesForBooleanArray(lenght2); + } + + static long ramBytesDiffForIntArrays(int length1, int lenght2) { + return ramBytesForIntArray(length1) - ramBytesForIntArray(lenght2); + } + + static long ramBytesDiffForLongArrays(int length1, int lenght2) { + return ramBytesForLongArray(length1) - ramBytesForLongArray(lenght2); + } + + static long ramBytesDiffForDoubleArrays(int length1, int lenght2) { + return ramBytesForDoubleArray(length1) - ramBytesForDoubleArray(lenght2); + } + + static long ramBytesForBooleanArray(int length) { + return alignObjectSize((long) Byte.BYTES * length); } - static long ramBytesForIntArray(int[] arr) { - return alignObjectSize((long) Integer.BYTES * arr.length); + static long ramBytesForIntArray(int length) { + return alignObjectSize((long) Integer.BYTES * length); } - static long ramBytesForLongArray(long[] arr) { - return alignObjectSize((long) Long.BYTES * arr.length); + static long ramBytesForLongArray(int length) { + return alignObjectSize((long) Long.BYTES * length); } - static long ramBytesForDoubleArray(double[] arr) { - return alignObjectSize((long) Long.BYTES * arr.length); + static long ramBytesForDoubleArray(int length) { + return alignObjectSize((long) Long.BYTES * length); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java index e794215a4c212..34e9e8274f076 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java @@ -109,7 +109,7 @@ public void testIntBlockBuilderWithPossiblyLargeEstimateEmpty() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newIntArrayBlock(new int[] {}, 0, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newIntArrayBlock(new int[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); } @@ -121,7 +121,7 @@ public void testIntBlockBuilderWithPossiblyLargeEstimateSingle() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newIntArrayBlock(new int[] { randomInt() }, 1, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newIntArrayBlock(new int[] { randomInt() }, 1, new int[] { 0, 1 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); @@ -199,7 +199,7 @@ public void testLongBlockBuilderWithPossiblyLargeEstimateEmpty() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newLongArrayBlock(new long[] {}, 0, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newLongArrayBlock(new long[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); } @@ -211,7 +211,7 @@ public void testLongBlockBuilderWithPossiblyLargeEstimateSingle() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newLongArrayBlock(new long[] { randomLong() }, 1, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newLongArrayBlock(new long[] { randomLong() }, 1, new int[] { 0, 1 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); @@ -289,7 +289,7 @@ public void testDoubleBlockBuilderWithPossiblyLargeEstimateEmpty() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newDoubleArrayBlock(new double[] {}, 0, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newDoubleArrayBlock(new double[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); } @@ -301,7 +301,7 @@ public void testDoubleBlockBuilderWithPossiblyLargeEstimateSingle() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newDoubleArrayBlock(new double[] { randomDouble() }, 1, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newDoubleArrayBlock(new double[] { randomDouble() }, 1, new int[] { 0, 1 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); @@ -379,7 +379,7 @@ public void testBooleanBlockBuilderWithPossiblyLargeEstimateEmpty() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newBooleanArrayBlock(new boolean[] {}, 0, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newBooleanArrayBlock(new boolean[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); } @@ -391,7 +391,7 @@ public void testBooleanBlockBuilderWithPossiblyLargeEstimateSingle() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newBooleanArrayBlock(new boolean[] { randomBoolean() }, 1, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newBooleanArrayBlock(new boolean[] { randomBoolean() }, 1, new int[] { 0, 1 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); @@ -470,7 +470,7 @@ public void testBytesRefBlockBuilderWithPossiblyLargeEstimateEmpty() { releaseAndAssertBreaker(block); var emptyArray = new BytesRefArray(0, bigArrays); - block = blockFactory.newBytesRefArrayBlock(emptyArray, 0, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newBytesRefArrayBlock(emptyArray, 0, new int[] { 0 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); } @@ -484,7 +484,7 @@ public void testBytesRefBlockBuilderWithPossiblyLargeEstimateSingle() { var array = new BytesRefArray(1, bigArrays); array.append(randomBytesRef()); - block = blockFactory.newBytesRefArrayBlock(array, 1, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newBytesRefArrayBlock(array, 1, new int[] { 0, 1 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BooleanBlockEqualityTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BooleanBlockEqualityTests.java index c0fc539cecc6c..3e2322200dcf0 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BooleanBlockEqualityTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BooleanBlockEqualityTests.java @@ -35,7 +35,7 @@ public void testEmptyBlock() { new BooleanArrayBlock( new boolean[] {}, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()), blockFactory @@ -43,7 +43,7 @@ public void testEmptyBlock() { new BooleanArrayBlock( new boolean[] { randomBoolean() }, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()), blockFactory diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BytesRefBlockEqualityTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BytesRefBlockEqualityTests.java index ec740db329c74..6b3fa5df9b9ff 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BytesRefBlockEqualityTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BytesRefBlockEqualityTests.java @@ -46,7 +46,7 @@ public void testEmptyBlock() { new BytesRefArrayBlock( bytesRefArray1, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()), blockFactory @@ -54,7 +54,7 @@ public void testEmptyBlock() { new BytesRefArrayBlock( bytesRefArray2, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()), blockFactory diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DoubleBlockEqualityTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DoubleBlockEqualityTests.java index e8f8fbcbf1c4c..c0a3b3b8ac751 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DoubleBlockEqualityTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DoubleBlockEqualityTests.java @@ -36,14 +36,14 @@ public void testEmptyBlock() { blockFactory.newDoubleArrayBlock( new double[] {}, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ), blockFactory.newDoubleArrayBlock( new double[] { 0 }, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ), diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/IntBlockEqualityTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/IntBlockEqualityTests.java index 6c1be6231e82c..5beb091cbfaca 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/IntBlockEqualityTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/IntBlockEqualityTests.java @@ -35,14 +35,14 @@ public void testEmptyBlock() { blockFactory.newIntArrayBlock( new int[] {}, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ), blockFactory.newIntArrayBlock( new int[] { 0 }, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ), diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/LongBlockEqualityTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/LongBlockEqualityTests.java index 27a2f9702a0ae..3e425439bb800 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/LongBlockEqualityTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/LongBlockEqualityTests.java @@ -35,14 +35,14 @@ public void testEmptyBlock() { blockFactory.newLongArrayBlock( new long[] {}, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ), blockFactory.newLongArrayBlock( new long[] { 0 }, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ),