From 8b861ca2df6746ecb75d875fc71f0c12ceb22ceb Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 22 Dec 2023 10:32:40 +0100 Subject: [PATCH 1/6] Correctly create blocks in tests Also make more correct assertions about block size. --- .../compute/data/BasicBlockTests.java | 21 ++--- .../compute/data/BlockAccountingTests.java | 85 +++++++++++++------ .../compute/data/BlockFactoryTests.java | 20 ++--- .../data/BooleanBlockEqualityTests.java | 4 +- .../data/BytesRefBlockEqualityTests.java | 4 +- .../data/DoubleBlockEqualityTests.java | 4 +- .../compute/data/IntBlockEqualityTests.java | 4 +- .../compute/data/LongBlockEqualityTests.java | 4 +- 8 files changed, 92 insertions(+), 54 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java index f2931a0d04a49..e53b9c5399578 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java @@ -60,25 +60,25 @@ public void testEmpty() { } void testEmpty(BlockFactory bf) { - assertZeroPositionsAndRelease(bf.newIntArrayBlock(new int[] {}, 0, new int[] {}, new BitSet(), randomOrdering())); + assertZeroPositionsAndRelease(bf.newIntArrayBlock(new int[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering())); assertZeroPositionsAndRelease(IntBlock.newBlockBuilder(0, bf).build()); assertZeroPositionsAndRelease(bf.newIntArrayVector(new int[] {}, 0)); assertZeroPositionsAndRelease(IntVector.newVectorBuilder(0, bf).build()); - assertZeroPositionsAndRelease(bf.newLongArrayBlock(new long[] {}, 0, new int[] {}, new BitSet(), randomOrdering())); + assertZeroPositionsAndRelease(bf.newLongArrayBlock(new long[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering())); assertZeroPositionsAndRelease(LongBlock.newBlockBuilder(0, bf).build()); assertZeroPositionsAndRelease(bf.newLongArrayVector(new long[] {}, 0)); assertZeroPositionsAndRelease(LongVector.newVectorBuilder(0, bf).build()); - assertZeroPositionsAndRelease(bf.newDoubleArrayBlock(new double[] {}, 0, new int[] {}, new BitSet(), randomOrdering())); + assertZeroPositionsAndRelease(bf.newDoubleArrayBlock(new double[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering())); assertZeroPositionsAndRelease(DoubleBlock.newBlockBuilder(0, bf).build()); assertZeroPositionsAndRelease(bf.newDoubleArrayVector(new double[] {}, 0)); assertZeroPositionsAndRelease(DoubleVector.newVectorBuilder(0, bf).build()); assertZeroPositionsAndRelease( - bf.newBytesRefArrayBlock(new BytesRefArray(0, bf.bigArrays()), 0, new int[] {}, new BitSet(), randomOrdering()) + bf.newBytesRefArrayBlock(new BytesRefArray(0, bf.bigArrays()), 0, new int[] { 0 }, new BitSet(), randomOrdering()) ); assertZeroPositionsAndRelease(BytesRefBlock.newBlockBuilder(0, bf).build()); assertZeroPositionsAndRelease(bf.newBytesRefArrayVector(new BytesRefArray(0, bf.bigArrays()), 0)); assertZeroPositionsAndRelease(BytesRefVector.newVectorBuilder(0, bf).build()); - assertZeroPositionsAndRelease(bf.newBooleanArrayBlock(new boolean[] {}, 0, new int[] {}, new BitSet(), randomOrdering())); + assertZeroPositionsAndRelease(bf.newBooleanArrayBlock(new boolean[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering())); assertZeroPositionsAndRelease(BooleanBlock.newBlockBuilder(0, bf).build()); assertZeroPositionsAndRelease(bf.newBooleanArrayVector(new boolean[] {}, 0)); assertZeroPositionsAndRelease(BooleanVector.newVectorBuilder(0, bf).build()); @@ -1266,13 +1266,14 @@ private Vector randomConstantVector() { private Block randomArrayBlock() { int positionCount = randomIntBetween(0, 100); int arrayType = randomIntBetween(0, 4); + int[] firstValueIndexes = IntStream.range(0, positionCount + 1).toArray(); return switch (arrayType) { case 0 -> { boolean[] values = new boolean[positionCount]; Arrays.fill(values, randomBoolean()); - yield blockFactory.newBooleanArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + yield blockFactory.newBooleanArrayBlock(values, positionCount, firstValueIndexes, new BitSet(), randomOrdering()); } case 1 -> { BytesRefArray values = new BytesRefArray(positionCount, BigArrays.NON_RECYCLING_INSTANCE); @@ -1280,25 +1281,25 @@ private Block randomArrayBlock() { values.append(new BytesRef(randomByteArrayOfLength(between(1, 20)))); } - yield blockFactory.newBytesRefArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + yield blockFactory.newBytesRefArrayBlock(values, positionCount, firstValueIndexes, new BitSet(), randomOrdering()); } case 2 -> { double[] values = new double[positionCount]; Arrays.fill(values, 1.0); - yield blockFactory.newDoubleArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + yield blockFactory.newDoubleArrayBlock(values, positionCount, firstValueIndexes, new BitSet(), randomOrdering()); } case 3 -> { int[] values = new int[positionCount]; Arrays.fill(values, 1); - yield blockFactory.newIntArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + yield blockFactory.newIntArrayBlock(values, positionCount, firstValueIndexes, new BitSet(), randomOrdering()); } default -> { long[] values = new long[positionCount]; Arrays.fill(values, 1L); - yield blockFactory.newLongArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + yield blockFactory.newLongArrayBlock(values, positionCount, firstValueIndexes, new BitSet(), randomOrdering()); } }; } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java index b5155f3199c1c..d34e639c32f0d 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java @@ -134,19 +134,22 @@ public void testBytesRefVector() { // Array Blocks public void testBooleanBlock() { BlockFactory blockFactory = blockFactory(); - Block empty = new BooleanArrayBlock(new boolean[] {}, 0, new int[0], null, Block.MvOrdering.UNORDERED, blockFactory); + Block empty = new BooleanArrayBlock(new boolean[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory); long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR); assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); Block emptyPlusOne = new BooleanArrayBlock( new boolean[] { randomBoolean() }, 1, - new int[] { 0 }, + new int[] { 0, 1 }, null, Block.MvOrdering.UNORDERED, blockFactory ); - assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + 1) + alignObjectSize(Integer.BYTES))); + assertThat( + emptyPlusOne.ramBytesUsed(), + is(empty.ramBytesUsed() + ramBytesDiffForBooleanArrays(1, 0) + ramBytesDiffForIntArrays(2, 1)) + ); boolean[] randomData = new boolean[randomIntBetween(2, 1024)]; int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); @@ -158,7 +161,10 @@ public void testBooleanBlock() { Block.MvOrdering.UNORDERED, blockFactory ); - long expected = empty.ramBytesUsed() + ramBytesForBooleanArray(randomData) + ramBytesForIntArray(valueIndices); + long expected = empty.ramBytesUsed() + ramBytesDiffForBooleanArrays(randomData.length, 0) + ramBytesDiffForIntArrays( + valueIndices.length, + 1 + ); assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); @@ -181,19 +187,19 @@ public void testBooleanBlockWithNullFirstValues() { public void testIntBlock() { BlockFactory blockFactory = blockFactory(); - Block empty = new IntArrayBlock(new int[] {}, 0, new int[] {}, null, Block.MvOrdering.UNORDERED, blockFactory); + Block empty = new IntArrayBlock(new int[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory); long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR); assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); Block emptyPlusOne = new IntArrayBlock( new int[] { randomInt() }, 1, - new int[] { 0 }, + new int[] { 0, 1 }, null, Block.MvOrdering.UNORDERED, blockFactory ); - assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + alignObjectSize(Integer.BYTES) + alignObjectSize(Integer.BYTES))); + assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + ramBytesDiffForIntArrays(1, 0) + ramBytesDiffForIntArrays(2, 1))); int[] randomData = new int[randomIntBetween(2, 1024)]; int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); @@ -205,7 +211,10 @@ public void testIntBlock() { Block.MvOrdering.UNORDERED, blockFactory ); - long expected = empty.ramBytesUsed() + ramBytesForIntArray(randomData) + ramBytesForIntArray(valueIndices); + long expected = empty.ramBytesUsed() + ramBytesDiffForIntArrays(randomData.length, 0) + ramBytesDiffForIntArrays( + valueIndices.length, + 1 + ); assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); @@ -222,19 +231,22 @@ public void testIntBlockWithNullFirstValues() { public void testLongBlock() { BlockFactory blockFactory = blockFactory(); - Block empty = new LongArrayBlock(new long[] {}, 0, new int[0], null, Block.MvOrdering.UNORDERED, blockFactory); + Block empty = new LongArrayBlock(new long[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory); long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR); assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); Block emptyPlusOne = new LongArrayBlock( new long[] { randomInt() }, 1, - new int[] { 0 }, + new int[] { 0, 1 }, null, Block.MvOrdering.UNORDERED, blockFactory ); - assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Long.BYTES) + alignObjectSize(Integer.BYTES))); + assertThat( + emptyPlusOne.ramBytesUsed(), + is(empty.ramBytesUsed() + ramBytesDiffForLongArrays(1, 0) + ramBytesDiffForIntArrays(2, 1)) + ); long[] randomData = new long[randomIntBetween(2, 1024)]; int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); @@ -246,7 +258,10 @@ public void testLongBlock() { Block.MvOrdering.UNORDERED, blockFactory ); - long expected = empty.ramBytesUsed() + ramBytesForLongArray(randomData) + ramBytesForIntArray(valueIndices); + long expected = empty.ramBytesUsed() + ramBytesDiffForLongArrays(randomData.length, 0) + ramBytesDiffForIntArrays( + valueIndices.length, + 1 + ); assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); @@ -269,19 +284,22 @@ public void testLongBlockWithNullFirstValues() { public void testDoubleBlock() { BlockFactory blockFactory = blockFactory(); - Block empty = new DoubleArrayBlock(new double[] {}, 0, new int[0], null, Block.MvOrdering.UNORDERED, blockFactory); + Block empty = new DoubleArrayBlock(new double[] {}, 0, new int[] { 0 }, null, Block.MvOrdering.UNORDERED, blockFactory); long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR); assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); Block emptyPlusOne = new DoubleArrayBlock( new double[] { randomInt() }, 1, - new int[] { 0 }, + new int[] { 0, 1 }, null, Block.MvOrdering.UNORDERED, blockFactory ); - assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Double.BYTES) + alignObjectSize(Integer.BYTES))); + assertThat( + emptyPlusOne.ramBytesUsed(), + is(empty.ramBytesUsed() + ramBytesDiffForDoubleArrays(1, 0) + ramBytesDiffForIntArrays(2, 1)) + ); double[] randomData = new double[randomIntBetween(2, 1024)]; int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); @@ -293,7 +311,10 @@ public void testDoubleBlock() { Block.MvOrdering.UNORDERED, blockFactory ); - long expected = empty.ramBytesUsed() + ramBytesForDoubleArray(randomData) + ramBytesForIntArray(valueIndices); + long expected = empty.ramBytesUsed() + ramBytesDiffForDoubleArrays(randomData.length, 0) + ramBytesDiffForIntArrays( + valueIndices.length, + 1 + ); assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); @@ -344,19 +365,35 @@ public long accumulateObject(Object o, long shallowSize, Map fiel } } - static long ramBytesForBooleanArray(boolean[] arr) { - return alignObjectSize((long) Byte.BYTES * arr.length); + static long ramBytesDiffForBooleanArrays(int length1, int lenght2) { + return ramBytesForBooleanArray(length1) - ramBytesForBooleanArray(lenght2); + } + + static long ramBytesDiffForIntArrays(int length1, int lenght2) { + return ramBytesForIntArray(length1) - ramBytesForIntArray(lenght2); + } + + static long ramBytesDiffForLongArrays(int length1, int lenght2) { + return ramBytesForLongArray(length1) - ramBytesForLongArray(lenght2); + } + + static long ramBytesDiffForDoubleArrays(int length1, int lenght2) { + return ramBytesForDoubleArray(length1) - ramBytesForDoubleArray(lenght2); + } + + static long ramBytesForBooleanArray(int length) { + return alignObjectSize((long) Byte.BYTES * length); } - static long ramBytesForIntArray(int[] arr) { - return alignObjectSize((long) Integer.BYTES * arr.length); + static long ramBytesForIntArray(int length) { + return alignObjectSize((long) Integer.BYTES * length); } - static long ramBytesForLongArray(long[] arr) { - return alignObjectSize((long) Long.BYTES * arr.length); + static long ramBytesForLongArray(int length) { + return alignObjectSize((long) Long.BYTES * length); } - static long ramBytesForDoubleArray(double[] arr) { - return alignObjectSize((long) Long.BYTES * arr.length); + static long ramBytesForDoubleArray(int length) { + return alignObjectSize((long) Long.BYTES * length); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java index 3a8500ea90cb3..ad73c093aa06a 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java @@ -109,7 +109,7 @@ public void testIntBlockBuilderWithPossiblyLargeEstimateEmpty() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newIntArrayBlock(new int[] {}, 0, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newIntArrayBlock(new int[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); } @@ -121,7 +121,7 @@ public void testIntBlockBuilderWithPossiblyLargeEstimateSingle() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newIntArrayBlock(new int[] { randomInt() }, 1, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newIntArrayBlock(new int[] { randomInt() }, 1, new int[] { 0, 1 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); @@ -199,7 +199,7 @@ public void testLongBlockBuilderWithPossiblyLargeEstimateEmpty() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newLongArrayBlock(new long[] {}, 0, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newLongArrayBlock(new long[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); } @@ -211,7 +211,7 @@ public void testLongBlockBuilderWithPossiblyLargeEstimateSingle() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newLongArrayBlock(new long[] { randomLong() }, 1, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newLongArrayBlock(new long[] { randomLong() }, 1, new int[] { 0, 1 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); @@ -289,7 +289,7 @@ public void testDoubleBlockBuilderWithPossiblyLargeEstimateEmpty() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newDoubleArrayBlock(new double[] {}, 0, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newDoubleArrayBlock(new double[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); } @@ -301,7 +301,7 @@ public void testDoubleBlockBuilderWithPossiblyLargeEstimateSingle() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newDoubleArrayBlock(new double[] { randomDouble() }, 1, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newDoubleArrayBlock(new double[] { randomDouble() }, 1, new int[] { 0, 1 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); @@ -379,7 +379,7 @@ public void testBooleanBlockBuilderWithPossiblyLargeEstimateEmpty() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newBooleanArrayBlock(new boolean[] {}, 0, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newBooleanArrayBlock(new boolean[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); } @@ -391,7 +391,7 @@ public void testBooleanBlockBuilderWithPossiblyLargeEstimateSingle() { var block = builder.build(); releaseAndAssertBreaker(block); - block = blockFactory.newBooleanArrayBlock(new boolean[] { randomBoolean() }, 1, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newBooleanArrayBlock(new boolean[] { randomBoolean() }, 1, new int[] { 0, 1 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); @@ -470,7 +470,7 @@ public void testBytesRefBlockBuilderWithPossiblyLargeEstimateEmpty() { releaseAndAssertBreaker(block); var emptyArray = new BytesRefArray(0, bigArrays); - block = blockFactory.newBytesRefArrayBlock(emptyArray, 0, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newBytesRefArrayBlock(emptyArray, 0, new int[] { 0 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); } @@ -484,7 +484,7 @@ public void testBytesRefBlockBuilderWithPossiblyLargeEstimateSingle() { var array = new BytesRefArray(1, bigArrays); array.append(randomBytesRef()); - block = blockFactory.newBytesRefArrayBlock(array, 1, new int[] {}, new BitSet(), randomOrdering()); + block = blockFactory.newBytesRefArrayBlock(array, 1, new int[] { 0, 1 }, new BitSet(), randomOrdering()); assertThat(breaker.getUsed(), greaterThan(0L)); releaseAndAssertBreaker(block); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BooleanBlockEqualityTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BooleanBlockEqualityTests.java index c0fc539cecc6c..3e2322200dcf0 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BooleanBlockEqualityTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BooleanBlockEqualityTests.java @@ -35,7 +35,7 @@ public void testEmptyBlock() { new BooleanArrayBlock( new boolean[] {}, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()), blockFactory @@ -43,7 +43,7 @@ public void testEmptyBlock() { new BooleanArrayBlock( new boolean[] { randomBoolean() }, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()), blockFactory diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BytesRefBlockEqualityTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BytesRefBlockEqualityTests.java index ec740db329c74..6b3fa5df9b9ff 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BytesRefBlockEqualityTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BytesRefBlockEqualityTests.java @@ -46,7 +46,7 @@ public void testEmptyBlock() { new BytesRefArrayBlock( bytesRefArray1, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()), blockFactory @@ -54,7 +54,7 @@ public void testEmptyBlock() { new BytesRefArrayBlock( bytesRefArray2, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()), blockFactory diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DoubleBlockEqualityTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DoubleBlockEqualityTests.java index e8f8fbcbf1c4c..c0a3b3b8ac751 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DoubleBlockEqualityTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DoubleBlockEqualityTests.java @@ -36,14 +36,14 @@ public void testEmptyBlock() { blockFactory.newDoubleArrayBlock( new double[] {}, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ), blockFactory.newDoubleArrayBlock( new double[] { 0 }, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ), diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/IntBlockEqualityTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/IntBlockEqualityTests.java index 6c1be6231e82c..5beb091cbfaca 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/IntBlockEqualityTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/IntBlockEqualityTests.java @@ -35,14 +35,14 @@ public void testEmptyBlock() { blockFactory.newIntArrayBlock( new int[] {}, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ), blockFactory.newIntArrayBlock( new int[] { 0 }, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ), diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/LongBlockEqualityTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/LongBlockEqualityTests.java index 27a2f9702a0ae..3e425439bb800 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/LongBlockEqualityTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/LongBlockEqualityTests.java @@ -35,14 +35,14 @@ public void testEmptyBlock() { blockFactory.newLongArrayBlock( new long[] {}, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ), blockFactory.newLongArrayBlock( new long[] { 0 }, 0, - new int[] {}, + new int[] { 0 }, BitSet.valueOf(new byte[] { 0b00 }), randomFrom(Block.MvOrdering.values()) ), From 4c78428a5cd5c171d4ed54f62036ad26962b6f8b Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 22 Dec 2023 11:06:07 +0100 Subject: [PATCH 2/6] Shallow copy in expand without nulls --- .../elasticsearch/compute/data/BooleanArrayBlock.java | 7 ++++++- .../compute/data/BooleanBigArrayBlock.java | 7 ++++++- .../compute/data/BytesRefArrayBlock.java | 7 ++++++- .../elasticsearch/compute/data/DoubleArrayBlock.java | 7 ++++++- .../compute/data/DoubleBigArrayBlock.java | 7 ++++++- .../org/elasticsearch/compute/data/IntArrayBlock.java | 7 ++++++- .../elasticsearch/compute/data/IntBigArrayBlock.java | 7 ++++++- .../elasticsearch/compute/data/LongArrayBlock.java | 7 ++++++- .../elasticsearch/compute/data/LongBigArrayBlock.java | 7 ++++++- .../elasticsearch/compute/data/X-ArrayBlock.java.st | 11 ++++++----- .../compute/data/X-BigArrayBlock.java.st | 7 ++++++- 11 files changed, 66 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java index f9b8358faee6b..60112ad528a03 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java @@ -31,7 +31,8 @@ final class BooleanArrayBlock extends AbstractArrayBlock implements BooleanBlock BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new BooleanArrayVector(values, values.length, blockFactory); + int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; + this.vector = new BooleanArrayVector(values, vectorLength, blockFactory); } @Override @@ -79,6 +80,10 @@ public BooleanBlock expand() { incRef(); return this; } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } // TODO use reference counting to share the vector try (var builder = blockFactory().newBooleanBlockBuilder(firstValueIndexes[getPositionCount()])) { for (int pos = 0; pos < getPositionCount(); pos++) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java index 17ed741bd59da..3952971967736 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java @@ -32,7 +32,8 @@ public BooleanBigArrayBlock( BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new BooleanBigArrayVector(values, (int) values.size(), blockFactory); + int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; + this.vector = new BooleanBigArrayVector(values, vectorLength, blockFactory); } @Override @@ -80,6 +81,10 @@ public BooleanBlock expand() { incRef(); return this; } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } // TODO use reference counting to share the vector try (var builder = blockFactory().newBooleanBlockBuilder(firstValueIndexes[getPositionCount()])) { for (int pos = 0; pos < getPositionCount(); pos++) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java index e2598d3d86b8f..7cc96f1cbb5c6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java @@ -34,7 +34,8 @@ final class BytesRefArrayBlock extends AbstractArrayBlock implements BytesRefBlo BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new BytesRefArrayVector(values, (int) values.size(), blockFactory); + int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; + this.vector = new BytesRefArrayVector(values, vectorLength, blockFactory); } @Override @@ -83,6 +84,10 @@ public BytesRefBlock expand() { incRef(); return this; } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } // TODO use reference counting to share the vector final BytesRef scratch = new BytesRef(); try (var builder = blockFactory().newBytesRefBlockBuilder(firstValueIndexes[getPositionCount()])) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java index 96e96ac459a50..e288c480503ca 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java @@ -31,7 +31,8 @@ final class DoubleArrayBlock extends AbstractArrayBlock implements DoubleBlock { BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new DoubleArrayVector(values, values.length, blockFactory); + int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; + this.vector = new DoubleArrayVector(values, vectorLength, blockFactory); } @Override @@ -79,6 +80,10 @@ public DoubleBlock expand() { incRef(); return this; } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } // TODO use reference counting to share the vector try (var builder = blockFactory().newDoubleBlockBuilder(firstValueIndexes[getPositionCount()])) { for (int pos = 0; pos < getPositionCount(); pos++) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java index 5b1dcbfc9d728..5b6a885e29308 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java @@ -32,7 +32,8 @@ public DoubleBigArrayBlock( BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new DoubleBigArrayVector(values, (int) values.size(), blockFactory); + int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; + this.vector = new DoubleBigArrayVector(values, vectorLength, blockFactory); } @Override @@ -80,6 +81,10 @@ public DoubleBlock expand() { incRef(); return this; } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } // TODO use reference counting to share the vector try (var builder = blockFactory().newDoubleBlockBuilder(firstValueIndexes[getPositionCount()])) { for (int pos = 0; pos < getPositionCount(); pos++) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java index e8f10ced11adc..108a5326a411c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java @@ -31,7 +31,8 @@ final class IntArrayBlock extends AbstractArrayBlock implements IntBlock { BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new IntArrayVector(values, values.length, blockFactory); + int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; + this.vector = new IntArrayVector(values, vectorLength, blockFactory); } @Override @@ -79,6 +80,10 @@ public IntBlock expand() { incRef(); return this; } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } // TODO use reference counting to share the vector try (var builder = blockFactory().newIntBlockBuilder(firstValueIndexes[getPositionCount()])) { for (int pos = 0; pos < getPositionCount(); pos++) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java index ad6033fb452a0..7eef12b7f4e3f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java @@ -32,7 +32,8 @@ public IntBigArrayBlock( BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new IntBigArrayVector(values, (int) values.size(), blockFactory); + int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; + this.vector = new IntBigArrayVector(values, vectorLength, blockFactory); } @Override @@ -80,6 +81,10 @@ public IntBlock expand() { incRef(); return this; } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } // TODO use reference counting to share the vector try (var builder = blockFactory().newIntBlockBuilder(firstValueIndexes[getPositionCount()])) { for (int pos = 0; pos < getPositionCount(); pos++) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java index 792f9b267e748..1e4eaeefbfd7f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java @@ -31,7 +31,8 @@ final class LongArrayBlock extends AbstractArrayBlock implements LongBlock { BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new LongArrayVector(values, values.length, blockFactory); + int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; + this.vector = new LongArrayVector(values, vectorLength, blockFactory); } @Override @@ -79,6 +80,10 @@ public LongBlock expand() { incRef(); return this; } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } // TODO use reference counting to share the vector try (var builder = blockFactory().newLongBlockBuilder(firstValueIndexes[getPositionCount()])) { for (int pos = 0; pos < getPositionCount(); pos++) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java index dc19a4038a9e9..4a6dceae0bac0 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java @@ -32,7 +32,8 @@ public LongBigArrayBlock( BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new LongBigArrayVector(values, (int) values.size(), blockFactory); + int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; + this.vector = new LongBigArrayVector(values, vectorLength, blockFactory); } @Override @@ -80,6 +81,10 @@ public LongBlock expand() { incRef(); return this; } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } // TODO use reference counting to share the vector try (var builder = blockFactory().newLongBlockBuilder(firstValueIndexes[getPositionCount()])) { for (int pos = 0; pos < getPositionCount(); pos++) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st index 01a6d70d63795..6d89b1d3ee412 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st @@ -42,11 +42,8 @@ final class $Type$ArrayBlock extends AbstractArrayBlock implements $Type$Block { BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - $if(BytesRef)$ - this.vector = new BytesRefArrayVector(values, (int) values.size(), blockFactory); - $else$ - this.vector = new $Type$ArrayVector(values, values.length, blockFactory); - $endif$ + int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; + this.vector = new $Type$ArrayVector(values, vectorLength, blockFactory); } @Override @@ -102,6 +99,10 @@ $endif$ incRef(); return this; } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } // TODO use reference counting to share the vector $if(BytesRef)$ final BytesRef scratch = new BytesRef(); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st index 989f119bca062..6fb4750f12dae 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st @@ -32,7 +32,8 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - this.vector = new $Type$BigArrayVector(values, (int) values.size(), blockFactory); + int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; + this.vector = new $Type$BigArrayVector(values, vectorLength, blockFactory); } @Override @@ -80,6 +81,10 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty incRef(); return this; } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } // TODO use reference counting to share the vector try (var builder = blockFactory().new$Type$BlockBuilder(firstValueIndexes[getPositionCount()])) { for (int pos = 0; pos < getPositionCount(); pos++) { From c3cf9195ff27738e49352ccffbdbd9ea93f46f5a Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 22 Dec 2023 12:20:51 +0100 Subject: [PATCH 3/6] Shallow copy when expanding with nulls --- .../compute/data/BooleanArrayBlock.java | 49 ++++++++++------ .../compute/data/BooleanBigArrayBlock.java | 50 +++++++++++------ .../compute/data/BytesRefArrayBlock.java | 50 ++++++++++------- .../compute/data/DoubleArrayBlock.java | 49 ++++++++++------ .../compute/data/DoubleBigArrayBlock.java | 50 +++++++++++------ .../compute/data/IntArrayBlock.java | 49 ++++++++++------ .../compute/data/IntBigArrayBlock.java | 50 +++++++++++------ .../compute/data/LongArrayBlock.java | 49 ++++++++++------ .../compute/data/LongBigArrayBlock.java | 50 +++++++++++------ .../compute/data/X-ArrayBlock.java.st | 56 ++++++++++--------- .../compute/data/X-BigArrayBlock.java.st | 50 +++++++++++------ 11 files changed, 346 insertions(+), 206 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java index 60112ad528a03..d085ec3c782b7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java @@ -29,10 +29,27 @@ final class BooleanArrayBlock extends AbstractArrayBlock implements BooleanBlock BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new BooleanArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private BooleanArrayBlock( + BooleanArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; - this.vector = new BooleanArrayVector(values, vectorLength, blockFactory); + this.vector = vector; } @Override @@ -47,6 +64,7 @@ public boolean getBoolean(int valueIndex) { @Override public BooleanBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newBooleanBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -80,25 +98,20 @@ public BooleanBlock expand() { incRef(); return this; } + vector.incRef(); if (nullsMask == null) { - vector.incRef(); return vector.asBlock(); } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newBooleanBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendBoolean(getBoolean(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); - } + BooleanArrayBlock expanded = new BooleanArrayBlock( + vector, + vector.getPositionCount(), + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java index 3952971967736..e0c828642eb71 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java @@ -30,10 +30,27 @@ public BooleanBigArrayBlock( BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new BooleanBigArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private BooleanBigArrayBlock( + BooleanBigArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; - this.vector = new BooleanBigArrayVector(values, vectorLength, blockFactory); + this.vector = vector; } @Override @@ -48,6 +65,7 @@ public boolean getBoolean(int valueIndex) { @Override public BooleanBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newBooleanBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -81,25 +99,21 @@ public BooleanBlock expand() { incRef(); return this; } + vector.incRef(); if (nullsMask == null) { - vector.incRef(); return vector.asBlock(); } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newBooleanBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendBoolean(getBoolean(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); - } + BooleanBigArrayBlock expanded = new BooleanBigArrayBlock( + vector, + vector.getPositionCount(), + null, + // TODO: we probably need to adjust the breaker before computing the shifted null mask + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java index 7cc96f1cbb5c6..fbc69dc1ac4e8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java @@ -32,10 +32,27 @@ final class BytesRefArrayBlock extends AbstractArrayBlock implements BytesRefBlo BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new BytesRefArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private BytesRefArrayBlock( + BytesRefArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; - this.vector = new BytesRefArrayVector(values, vectorLength, blockFactory); + this.vector = vector; } @Override @@ -50,6 +67,7 @@ public BytesRef getBytesRef(int valueIndex, BytesRef dest) { @Override public BytesRefBlock filter(int... positions) { + // TODO use reference counting to share the vector final BytesRef scratch = new BytesRef(); try (var builder = blockFactory().newBytesRefBlockBuilder(positions.length)) { for (int pos : positions) { @@ -84,26 +102,20 @@ public BytesRefBlock expand() { incRef(); return this; } + vector.incRef(); if (nullsMask == null) { - vector.incRef(); return vector.asBlock(); } - // TODO use reference counting to share the vector - final BytesRef scratch = new BytesRef(); - try (var builder = blockFactory().newBytesRefBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendBytesRef(getBytesRef(i, scratch)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); - } + BytesRefArrayBlock expanded = new BytesRefArrayBlock( + vector, + vector.getPositionCount(), + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java index e288c480503ca..974ec4d214ab3 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java @@ -29,10 +29,27 @@ final class DoubleArrayBlock extends AbstractArrayBlock implements DoubleBlock { BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new DoubleArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private DoubleArrayBlock( + DoubleArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; - this.vector = new DoubleArrayVector(values, vectorLength, blockFactory); + this.vector = vector; } @Override @@ -47,6 +64,7 @@ public double getDouble(int valueIndex) { @Override public DoubleBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newDoubleBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -80,25 +98,20 @@ public DoubleBlock expand() { incRef(); return this; } + vector.incRef(); if (nullsMask == null) { - vector.incRef(); return vector.asBlock(); } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newDoubleBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendDouble(getDouble(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); - } + DoubleArrayBlock expanded = new DoubleArrayBlock( + vector, + vector.getPositionCount(), + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java index 5b6a885e29308..3602c0515f8a6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java @@ -30,10 +30,27 @@ public DoubleBigArrayBlock( BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new DoubleBigArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private DoubleBigArrayBlock( + DoubleBigArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; - this.vector = new DoubleBigArrayVector(values, vectorLength, blockFactory); + this.vector = vector; } @Override @@ -48,6 +65,7 @@ public double getDouble(int valueIndex) { @Override public DoubleBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newDoubleBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -81,25 +99,21 @@ public DoubleBlock expand() { incRef(); return this; } + vector.incRef(); if (nullsMask == null) { - vector.incRef(); return vector.asBlock(); } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newDoubleBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendDouble(getDouble(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); - } + DoubleBigArrayBlock expanded = new DoubleBigArrayBlock( + vector, + vector.getPositionCount(), + null, + // TODO: we probably need to adjust the breaker before computing the shifted null mask + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java index 108a5326a411c..9f68288ee0fc9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java @@ -29,10 +29,27 @@ final class IntArrayBlock extends AbstractArrayBlock implements IntBlock { BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new IntArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private IntArrayBlock( + IntArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; - this.vector = new IntArrayVector(values, vectorLength, blockFactory); + this.vector = vector; } @Override @@ -47,6 +64,7 @@ public int getInt(int valueIndex) { @Override public IntBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newIntBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -80,25 +98,20 @@ public IntBlock expand() { incRef(); return this; } + vector.incRef(); if (nullsMask == null) { - vector.incRef(); return vector.asBlock(); } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newIntBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendInt(getInt(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); - } + IntArrayBlock expanded = new IntArrayBlock( + vector, + vector.getPositionCount(), + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java index 7eef12b7f4e3f..f12e6fdd3347f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java @@ -30,10 +30,27 @@ public IntBigArrayBlock( BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new IntBigArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private IntBigArrayBlock( + IntBigArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; - this.vector = new IntBigArrayVector(values, vectorLength, blockFactory); + this.vector = vector; } @Override @@ -48,6 +65,7 @@ public int getInt(int valueIndex) { @Override public IntBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newIntBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -81,25 +99,21 @@ public IntBlock expand() { incRef(); return this; } + vector.incRef(); if (nullsMask == null) { - vector.incRef(); return vector.asBlock(); } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newIntBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendInt(getInt(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); - } + IntBigArrayBlock expanded = new IntBigArrayBlock( + vector, + vector.getPositionCount(), + null, + // TODO: we probably need to adjust the breaker before computing the shifted null mask + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java index 1e4eaeefbfd7f..da28ad50ab9c7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java @@ -29,10 +29,27 @@ final class LongArrayBlock extends AbstractArrayBlock implements LongBlock { BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new LongArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private LongArrayBlock( + LongArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; - this.vector = new LongArrayVector(values, vectorLength, blockFactory); + this.vector = vector; } @Override @@ -47,6 +64,7 @@ public long getLong(int valueIndex) { @Override public LongBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newLongBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -80,25 +98,20 @@ public LongBlock expand() { incRef(); return this; } + vector.incRef(); if (nullsMask == null) { - vector.incRef(); return vector.asBlock(); } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newLongBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendLong(getLong(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); - } + LongArrayBlock expanded = new LongArrayBlock( + vector, + vector.getPositionCount(), + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java index 4a6dceae0bac0..ba7656d180217 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java @@ -30,10 +30,27 @@ public LongBigArrayBlock( BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new LongBigArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private LongBigArrayBlock( + LongBigArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; - this.vector = new LongBigArrayVector(values, vectorLength, blockFactory); + this.vector = vector; } @Override @@ -48,6 +65,7 @@ public long getLong(int valueIndex) { @Override public LongBlock filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().newLongBlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -81,25 +99,21 @@ public LongBlock expand() { incRef(); return this; } + vector.incRef(); if (nullsMask == null) { - vector.incRef(); return vector.asBlock(); } - // TODO use reference counting to share the vector - try (var builder = blockFactory().newLongBlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.appendLong(getLong(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); - } + LongBigArrayBlock expanded = new LongBigArrayBlock( + vector, + vector.getPositionCount(), + null, + // TODO: we probably need to adjust the breaker before computing the shifted null mask + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st index 6d89b1d3ee412..9e573ac13df3f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st @@ -40,10 +40,27 @@ final class $Type$ArrayBlock extends AbstractArrayBlock implements $Type$Block { BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new $Type$ArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private $Type$ArrayBlock( + $Type$ArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; - this.vector = new $Type$ArrayVector(values, vectorLength, blockFactory); + this.vector = vector; } @Override @@ -63,6 +80,7 @@ $endif$ @Override public $Type$Block filter(int... positions) { + // TODO use reference counting to share the vector $if(BytesRef)$ final BytesRef scratch = new BytesRef(); $endif$ @@ -99,32 +117,20 @@ $endif$ incRef(); return this; } + vector.incRef(); if (nullsMask == null) { - vector.incRef(); return vector.asBlock(); } - // TODO use reference counting to share the vector -$if(BytesRef)$ - final BytesRef scratch = new BytesRef(); -$endif$ - try (var builder = blockFactory().new$Type$BlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { -$if(BytesRef)$ - builder.append$Type$(get$Type$(i, scratch)); -$else$ - builder.append$Type$(get$Type$(i)); -$endif$ - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); - } + $Type$ArrayBlock expanded = new $Type$ArrayBlock( + vector, + vector.getPositionCount(), + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + return expanded; } private long ramBytesUsedOnlyBlock() { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st index 6fb4750f12dae..efc52242873ea 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st @@ -30,10 +30,27 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty BitSet nulls, MvOrdering mvOrdering, BlockFactory blockFactory + ) { + this( + new $Type$BigArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering, + blockFactory + ); + } + + private $Type$BigArrayBlock( + $Type$BigArrayVector vector, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory ) { super(positionCount, firstValueIndexes, nulls, mvOrdering, blockFactory); - int vectorLength = firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount]; - this.vector = new $Type$BigArrayVector(values, vectorLength, blockFactory); + this.vector = vector; } @Override @@ -48,6 +65,7 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty @Override public $Type$Block filter(int... positions) { + // TODO use reference counting to share the vector try (var builder = blockFactory().new$Type$BlockBuilder(positions.length)) { for (int pos : positions) { if (isNull(pos)) { @@ -81,25 +99,21 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty incRef(); return this; } + vector.incRef(); if (nullsMask == null) { - vector.incRef(); return vector.asBlock(); } - // TODO use reference counting to share the vector - try (var builder = blockFactory().new$Type$BlockBuilder(firstValueIndexes[getPositionCount()])) { - for (int pos = 0; pos < getPositionCount(); pos++) { - if (isNull(pos)) { - builder.appendNull(); - continue; - } - int first = getFirstValueIndex(pos); - int end = first + getValueCount(pos); - for (int i = first; i < end; i++) { - builder.append$Type$(get$Type$(i)); - } - } - return builder.mvOrdering(MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING).build(); - } + $Type$BigArrayBlock expanded = new $Type$BigArrayBlock( + vector, + vector.getPositionCount(), + null, + // TODO: we probably need to adjust the breaker before computing the shifted null mask + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + blockFactory() + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + return expanded; } private long ramBytesUsedOnlyBlock() { From ba9e9ea96bb1e75aeb7f578808d471ae77fcfbae Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 22 Dec 2023 15:07:08 +0100 Subject: [PATCH 4/6] Update docs/changelog/103681.yaml --- docs/changelog/103681.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/103681.yaml diff --git a/docs/changelog/103681.yaml b/docs/changelog/103681.yaml new file mode 100644 index 0000000000000..bba73c8e3a7d4 --- /dev/null +++ b/docs/changelog/103681.yaml @@ -0,0 +1,6 @@ +pr: 103681 +summary: "ESQL: Expand shallow copy with vecs" +area: ES|QL +type: enhancement +issues: + - 100528 From acd2ddc3c5c6f25f3f7bc9cb7dca37aa80332976 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 3 Jan 2024 12:39:11 +0100 Subject: [PATCH 5/6] Do not leak vecs if breaker trips Thanks @dnhatn for the remark. --- .../org/elasticsearch/compute/data/BooleanArrayBlock.java | 6 +++++- .../elasticsearch/compute/data/BooleanBigArrayBlock.java | 5 ++++- .../org/elasticsearch/compute/data/BytesRefArrayBlock.java | 6 +++++- .../org/elasticsearch/compute/data/DoubleArrayBlock.java | 6 +++++- .../org/elasticsearch/compute/data/DoubleBigArrayBlock.java | 5 ++++- .../org/elasticsearch/compute/data/IntArrayBlock.java | 6 +++++- .../org/elasticsearch/compute/data/IntBigArrayBlock.java | 5 ++++- .../org/elasticsearch/compute/data/LongArrayBlock.java | 6 +++++- .../org/elasticsearch/compute/data/LongBigArrayBlock.java | 5 ++++- .../org/elasticsearch/compute/data/X-ArrayBlock.java.st | 6 +++++- .../org/elasticsearch/compute/data/X-BigArrayBlock.java.st | 5 ++++- 11 files changed, 50 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java index d085ec3c782b7..cf2597dd4d75a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java @@ -98,19 +98,23 @@ public BooleanBlock expand() { incRef(); return this; } - vector.incRef(); if (nullsMask == null) { + vector.incRef(); return vector.asBlock(); } + BooleanArrayBlock expanded = new BooleanArrayBlock( vector, vector.getPositionCount(), null, + // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); return expanded; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java index e0c828642eb71..aa35aa1f876e3 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java @@ -99,10 +99,11 @@ public BooleanBlock expand() { incRef(); return this; } - vector.incRef(); if (nullsMask == null) { + vector.incRef(); return vector.asBlock(); } + BooleanBigArrayBlock expanded = new BooleanBigArrayBlock( vector, vector.getPositionCount(), @@ -113,6 +114,8 @@ public BooleanBlock expand() { blockFactory() ); blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); return expanded; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java index fbc69dc1ac4e8..12d46bb3162bc 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java @@ -102,19 +102,23 @@ public BytesRefBlock expand() { incRef(); return this; } - vector.incRef(); if (nullsMask == null) { + vector.incRef(); return vector.asBlock(); } + BytesRefArrayBlock expanded = new BytesRefArrayBlock( vector, vector.getPositionCount(), null, + // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); return expanded; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java index 974ec4d214ab3..1156fe5e8d05a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java @@ -98,19 +98,23 @@ public DoubleBlock expand() { incRef(); return this; } - vector.incRef(); if (nullsMask == null) { + vector.incRef(); return vector.asBlock(); } + DoubleArrayBlock expanded = new DoubleArrayBlock( vector, vector.getPositionCount(), null, + // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); return expanded; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java index 3602c0515f8a6..f38bc6e376a25 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java @@ -99,10 +99,11 @@ public DoubleBlock expand() { incRef(); return this; } - vector.incRef(); if (nullsMask == null) { + vector.incRef(); return vector.asBlock(); } + DoubleBigArrayBlock expanded = new DoubleBigArrayBlock( vector, vector.getPositionCount(), @@ -113,6 +114,8 @@ public DoubleBlock expand() { blockFactory() ); blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); return expanded; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java index 9f68288ee0fc9..590375fee0961 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java @@ -98,19 +98,23 @@ public IntBlock expand() { incRef(); return this; } - vector.incRef(); if (nullsMask == null) { + vector.incRef(); return vector.asBlock(); } + IntArrayBlock expanded = new IntArrayBlock( vector, vector.getPositionCount(), null, + // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); return expanded; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java index f12e6fdd3347f..953317669e2c8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java @@ -99,10 +99,11 @@ public IntBlock expand() { incRef(); return this; } - vector.incRef(); if (nullsMask == null) { + vector.incRef(); return vector.asBlock(); } + IntBigArrayBlock expanded = new IntBigArrayBlock( vector, vector.getPositionCount(), @@ -113,6 +114,8 @@ public IntBlock expand() { blockFactory() ); blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); return expanded; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java index da28ad50ab9c7..b5761dd257282 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java @@ -98,19 +98,23 @@ public LongBlock expand() { incRef(); return this; } - vector.incRef(); if (nullsMask == null) { + vector.incRef(); return vector.asBlock(); } + LongArrayBlock expanded = new LongArrayBlock( vector, vector.getPositionCount(), null, + // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); return expanded; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java index ba7656d180217..146fce261a5a7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java @@ -99,10 +99,11 @@ public LongBlock expand() { incRef(); return this; } - vector.incRef(); if (nullsMask == null) { + vector.incRef(); return vector.asBlock(); } + LongBigArrayBlock expanded = new LongBigArrayBlock( vector, vector.getPositionCount(), @@ -113,6 +114,8 @@ public LongBlock expand() { blockFactory() ); blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); return expanded; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st index 9e573ac13df3f..8c778239e77ff 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st @@ -117,19 +117,23 @@ $endif$ incRef(); return this; } - vector.incRef(); if (nullsMask == null) { + vector.incRef(); return vector.asBlock(); } + $Type$ArrayBlock expanded = new $Type$ArrayBlock( vector, vector.getPositionCount(), null, + // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); return expanded; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st index efc52242873ea..548a7468b6f5d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st @@ -99,10 +99,11 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty incRef(); return this; } - vector.incRef(); if (nullsMask == null) { + vector.incRef(); return vector.asBlock(); } + $Type$BigArrayBlock expanded = new $Type$BigArrayBlock( vector, vector.getPositionCount(), @@ -113,6 +114,8 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty blockFactory() ); blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); return expanded; } From 1dca0ad1bdfc4be2c1dad946f952938aa7d51ab3 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 3 Jan 2024 15:01:19 +0100 Subject: [PATCH 6/6] Estimate size before expanding --- .../elasticsearch/compute/data/BooleanArrayBlock.java | 10 +++++++--- .../compute/data/BooleanBigArrayBlock.java | 10 +++++++--- .../elasticsearch/compute/data/BytesRefArrayBlock.java | 10 +++++++--- .../elasticsearch/compute/data/DoubleArrayBlock.java | 10 +++++++--- .../compute/data/DoubleBigArrayBlock.java | 10 +++++++--- .../org/elasticsearch/compute/data/IntArrayBlock.java | 10 +++++++--- .../elasticsearch/compute/data/IntBigArrayBlock.java | 10 +++++++--- .../org/elasticsearch/compute/data/LongArrayBlock.java | 10 +++++++--- .../elasticsearch/compute/data/LongBigArrayBlock.java | 10 +++++++--- .../compute/data/BlockRamUsageEstimator.java | 6 +++++- .../elasticsearch/compute/data/X-ArrayBlock.java.st | 10 +++++++--- .../elasticsearch/compute/data/X-BigArrayBlock.java.st | 10 +++++++--- 12 files changed, 82 insertions(+), 34 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java index cf2597dd4d75a..d7d0856963019 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java @@ -103,16 +103,20 @@ public BooleanBlock expand() { return vector.asBlock(); } + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + BooleanArrayBlock expanded = new BooleanArrayBlock( vector, - vector.getPositionCount(), + expandedPositionCount, null, - // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); - blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. vector.incRef(); return expanded; diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java index aa35aa1f876e3..d75e988fe3a84 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayBlock.java @@ -104,16 +104,20 @@ public BooleanBlock expand() { return vector.asBlock(); } + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + BooleanBigArrayBlock expanded = new BooleanBigArrayBlock( vector, - vector.getPositionCount(), + expandedPositionCount, null, - // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); - blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. vector.incRef(); return expanded; diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java index 12d46bb3162bc..b236949e92018 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java @@ -107,16 +107,20 @@ public BytesRefBlock expand() { return vector.asBlock(); } + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + BytesRefArrayBlock expanded = new BytesRefArrayBlock( vector, - vector.getPositionCount(), + expandedPositionCount, null, - // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); - blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. vector.incRef(); return expanded; diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java index 1156fe5e8d05a..6089679904e48 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java @@ -103,16 +103,20 @@ public DoubleBlock expand() { return vector.asBlock(); } + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + DoubleArrayBlock expanded = new DoubleArrayBlock( vector, - vector.getPositionCount(), + expandedPositionCount, null, - // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); - blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. vector.incRef(); return expanded; diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java index f38bc6e376a25..adc09d5755b53 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayBlock.java @@ -104,16 +104,20 @@ public DoubleBlock expand() { return vector.asBlock(); } + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + DoubleBigArrayBlock expanded = new DoubleBigArrayBlock( vector, - vector.getPositionCount(), + expandedPositionCount, null, - // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); - blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. vector.incRef(); return expanded; diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java index 590375fee0961..1e5ae7a3de448 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java @@ -103,16 +103,20 @@ public IntBlock expand() { return vector.asBlock(); } + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + IntArrayBlock expanded = new IntArrayBlock( vector, - vector.getPositionCount(), + expandedPositionCount, null, - // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); - blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. vector.incRef(); return expanded; diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java index 953317669e2c8..068f550a56d5b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayBlock.java @@ -104,16 +104,20 @@ public IntBlock expand() { return vector.asBlock(); } + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + IntBigArrayBlock expanded = new IntBigArrayBlock( vector, - vector.getPositionCount(), + expandedPositionCount, null, - // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); - blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. vector.incRef(); return expanded; diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java index b5761dd257282..0d8464a95ed5b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java @@ -103,16 +103,20 @@ public LongBlock expand() { return vector.asBlock(); } + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + LongArrayBlock expanded = new LongArrayBlock( vector, - vector.getPositionCount(), + expandedPositionCount, null, - // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); - blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. vector.incRef(); return expanded; diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java index 146fce261a5a7..eb1a353352ac9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayBlock.java @@ -104,16 +104,20 @@ public LongBlock expand() { return vector.asBlock(); } + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + LongBigArrayBlock expanded = new LongBigArrayBlock( vector, - vector.getPositionCount(), + expandedPositionCount, null, - // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); - blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. vector.incRef(); return expanded; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java index bdc4dbef15bd2..d1f1bac940714 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java @@ -23,6 +23,10 @@ public static long sizeOf(@Nullable int[] arr) { /** Returns the size in bytes used by the bitset. Otherwise, returns 0 if null. Not exact, but good enough */ public static long sizeOfBitSet(@Nullable BitSet bitset) { - return bitset == null ? 0 : BITSET_BASE_RAM_USAGE + (bitset.size() / Byte.SIZE); + return bitset == null ? 0 : sizeOfBitSet(bitset.size()); + } + + public static long sizeOfBitSet(long size) { + return BITSET_BASE_RAM_USAGE + (size / Byte.SIZE); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st index 8c778239e77ff..245089ff2a83e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st @@ -122,16 +122,20 @@ $endif$ return vector.asBlock(); } + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + $Type$ArrayBlock expanded = new $Type$ArrayBlock( vector, - vector.getPositionCount(), + expandedPositionCount, null, - // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); - blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. vector.incRef(); return expanded; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st index 548a7468b6f5d..a5f5001802cb7 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayBlock.java.st @@ -104,16 +104,20 @@ public final class $Type$BigArrayBlock extends AbstractArrayBlock implements $Ty return vector.asBlock(); } + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount); + blockFactory().adjustBreaker(bitSetRamUsedEstimate, false); + $Type$BigArrayBlock expanded = new $Type$BigArrayBlock( vector, - vector.getPositionCount(), + expandedPositionCount, null, - // TODO: we probably need to adjust the breaker before computing the shifted null mask shiftNullsToExpandedPositions(), MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, blockFactory() ); - blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock(), true); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate, true); // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. vector.incRef(); return expanded;