diff --git a/docs/changelog/101488.yaml b/docs/changelog/101488.yaml new file mode 100644 index 0000000000000..1db48a63f8542 --- /dev/null +++ b/docs/changelog/101488.yaml @@ -0,0 +1,5 @@ +pr: 101488 +summary: "ESQL: More tracking in `BlockHash` impls" +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java index 3f696e3387c54..6f041a6681659 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BooleanBlockHash.java @@ -61,7 +61,7 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { private IntVector add(BooleanVector vector) { int positions = vector.getPositionCount(); - try (var builder = IntVector.newVectorFixedBuilder(positions, blockFactory)) { + try (var builder = blockFactory.newIntVectorFixedBuilder(positions)) { for (int i = 0; i < positions; i++) { builder.appendInt(MultivalueDedupeBoolean.hashOrd(everSeen, vector.getBoolean(i))); } @@ -75,28 +75,30 @@ private IntBlock add(BooleanBlock block) { @Override public BooleanBlock[] getKeys() { - BooleanBlock.Builder builder = BooleanBlock.newBlockBuilder(everSeen.length); - if (everSeen[NULL_ORD]) { - builder.appendNull(); - } - if (everSeen[FALSE_ORD]) { - builder.appendBoolean(false); - } - if (everSeen[TRUE_ORD]) { - builder.appendBoolean(true); + try (BooleanBlock.Builder builder = blockFactory.newBooleanBlockBuilder(everSeen.length)) { + if (everSeen[NULL_ORD]) { + builder.appendNull(); + } + if (everSeen[FALSE_ORD]) { + builder.appendBoolean(false); + } + if (everSeen[TRUE_ORD]) { + builder.appendBoolean(true); + } + return new BooleanBlock[] { builder.build() }; } - return new BooleanBlock[] { builder.build() }; } @Override public IntVector nonEmpty() { - IntVector.Builder builder = IntVector.newVectorBuilder(everSeen.length); - for (int i = 0; i < everSeen.length; i++) { - if (everSeen[i]) { - builder.appendInt(i); + try (IntVector.Builder builder = blockFactory.newIntVectorBuilder(everSeen.length)) { + for (int i = 0; i < everSeen.length; i++) { + if (everSeen[i]) { + builder.appendInt(i); + } } + return builder.build(); } - return builder.build(); } public BitArray seenGroupIds(BigArrays bigArrays) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java index 27e40e82f7df1..2f1bb4f858ff4 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRefBlockHash.java @@ -68,8 +68,8 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { addInput.add(0, groupIds); } } else { - try (IntBlock groupIds = add(bytesVector).asBlock()) { - addInput.add(0, groupIds.asVector()); + try (IntVector groupIds = add(bytesVector)) { + addInput.add(0, groupIds); } } } @@ -77,7 +77,7 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { private IntVector add(BytesRefVector vector) { int positions = vector.getPositionCount(); - try (var builder = IntVector.newVectorFixedBuilder(positions, blockFactory)) { + try (var builder = blockFactory.newIntVectorFixedBuilder(positions)) { for (int i = 0; i < positions; i++) { builder.appendInt(Math.toIntExact(hashOrdToGroupNullReserved(bytesRefHash.add(vector.getBytesRef(i, bytes))))); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRefLongBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRefLongBlockHash.java index 1721ded09849e..da2c85e532016 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRefLongBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRefLongBlockHash.java @@ -74,7 +74,9 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { if (vector1 != null && vector2 != null) { addInput.add(0, add(vector1, vector2)); } else { - new AddBlock(block1, block2, addInput).add(); + try (AddWork work = new AddWork(block1, block2, addInput)) { + work.add(); + } } } @@ -91,12 +93,12 @@ public IntVector add(BytesRefVector vector1, LongVector vector2) { private static final long[] EMPTY = new long[0]; - private class AddBlock extends LongLongBlockHash.AbstractAddBlock { + private class AddWork extends LongLongBlockHash.AbstractAddBlock { private final BytesRefBlock block1; private final LongBlock block2; - AddBlock(BytesRefBlock block1, LongBlock block2, GroupingAggregatorFunction.AddInput addInput) { - super(emitBatchSize, addInput); + AddWork(BytesRefBlock block1, LongBlock block2, GroupingAggregatorFunction.AddInput addInput) { + super(blockFactory, emitBatchSize, addInput); this.block1 = block1; this.block2 = block2; } @@ -165,18 +167,29 @@ void add() { @Override public Block[] getKeys() { int positions = (int) finalHash.size(); - BytesRefVector.Builder keys1 = BytesRefVector.newVectorBuilder(positions); - LongVector.Builder keys2 = LongVector.newVectorBuilder(positions); - BytesRef scratch = new BytesRef(); - for (long i = 0; i < positions; i++) { - keys2.appendLong(finalHash.getKey2(i)); - long h1 = finalHash.getKey1(i); - keys1.appendBytesRef(bytesHash.get(h1, scratch)); + BytesRefVector k1 = null; + LongVector k2 = null; + try ( + BytesRefVector.Builder keys1 = blockFactory.newBytesRefVectorBuilder(positions); + LongVector.Builder keys2 = blockFactory.newLongVectorBuilder(positions) + ) { + BytesRef scratch = new BytesRef(); + for (long i = 0; i < positions; i++) { + keys2.appendLong(finalHash.getKey2(i)); + long h1 = finalHash.getKey1(i); + keys1.appendBytesRef(bytesHash.get(h1, scratch)); + } + k1 = keys1.build(); + k2 = keys2.build(); + } finally { + if (k2 == null) { + Releasables.closeExpectNoException(k1); + } } if (reverseOutput) { - return new Block[] { keys2.build().asBlock(), keys1.build().asBlock() }; + return new Block[] { k2.asBlock(), k1.asBlock() }; } else { - return new Block[] { keys1.build().asBlock(), keys2.build().asBlock() }; + return new Block[] { k1.asBlock(), k2.asBlock() }; } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java index bbf5911521122..a8a67180775fb 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/DoubleBlockHash.java @@ -62,8 +62,8 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { addInput.add(0, groupIds); } } else { - try (IntBlock groupIds = add(doubleVector).asBlock()) { - addInput.add(0, groupIds.asVector()); + try (IntVector groupIds = add(doubleVector)) { + addInput.add(0, groupIds); } } } @@ -71,7 +71,7 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { private IntVector add(DoubleVector vector) { int positions = vector.getPositionCount(); - try (var builder = IntVector.newVectorFixedBuilder(positions, blockFactory)) { + try (var builder = blockFactory.newIntVectorFixedBuilder(positions)) { for (int i = 0; i < positions; i++) { builder.appendInt(Math.toIntExact(hashOrdToGroupNullReserved(longHash.add(Double.doubleToLongBits(vector.getDouble(i)))))); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java index 17c7fab73d2be..79e03e4dc0ed5 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/IntBlockHash.java @@ -59,8 +59,8 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { addInput.add(0, groupIds); } } else { - try (IntBlock groupIds = add(intVector).asBlock()) { - addInput.add(0, groupIds.asVector()); + try (IntVector groupIds = add(intVector)) { + addInput.add(0, groupIds); } } } @@ -68,7 +68,7 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { private IntVector add(IntVector vector) { int positions = vector.getPositionCount(); - try (var builder = IntVector.newVectorFixedBuilder(positions, blockFactory)) { + try (var builder = blockFactory.newIntVectorFixedBuilder(positions)) { for (int i = 0; i < positions; i++) { builder.appendInt(Math.toIntExact(hashOrdToGroupNullReserved(longHash.add(vector.getInt(i))))); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java index 1d3be57bb40d7..c736cfae65ee7 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/LongBlockHash.java @@ -62,8 +62,8 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { addInput.add(0, groupIds); } } else { - try (IntBlock groupIds = add(longVector).asBlock()) { // Ugh!! - addInput.add(0, groupIds.asVector()); + try (IntVector groupIds = add(longVector)) { + addInput.add(0, groupIds); } } } @@ -71,7 +71,7 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { private IntVector add(LongVector vector) { int positions = vector.getPositionCount(); - try (var builder = IntVector.newVectorFixedBuilder(positions, blockFactory)) { + try (var builder = blockFactory.newIntVectorFixedBuilder(positions)) { for (int i = 0; i < positions; i++) { builder.appendInt(Math.toIntExact(hashOrdToGroupNullReserved(longHash.add(vector.getLong(i))))); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/LongLongBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/LongLongBlockHash.java index f36e8c76ec98b..ce53f0bb8e7f4 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/LongLongBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/LongLongBlockHash.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction; import org.elasticsearch.compute.aggregation.SeenGroupIds; import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; @@ -74,13 +75,12 @@ private IntVector add(LongVector vector1, LongVector vector2) { private static final long[] EMPTY = new long[0]; - // TODO: this uses the non-breaking block factory - update to use this blockFactory - private class AddBlock extends AbstractAddBlock implements Releasable { + private class AddBlock extends AbstractAddBlock { private final LongBlock block1; private final LongBlock block2; AddBlock(LongBlock block1, LongBlock block2, GroupingAggregatorFunction.AddInput addInput) { - super(emitBatchSize, addInput); + super(blockFactory, emitBatchSize, addInput); this.block1 = block1; this.block2 = block2; } @@ -137,14 +137,10 @@ void add() { } emitOrds(); } - - @Override - public void close() { - Releasables.closeExpectNoException(block1, block2); - } } - static class AbstractAddBlock { + static class AbstractAddBlock implements Releasable { + private final BlockFactory blockFactory; private final int emitBatchSize; private final GroupingAggregatorFunction.AddInput addInput; @@ -152,11 +148,12 @@ static class AbstractAddBlock { private int added = 0; protected IntBlock.Builder ords; - AbstractAddBlock(int emitBatchSize, GroupingAggregatorFunction.AddInput addInput) { + AbstractAddBlock(BlockFactory blockFactory, int emitBatchSize, GroupingAggregatorFunction.AddInput addInput) { + this.blockFactory = blockFactory; this.emitBatchSize = emitBatchSize; this.addInput = addInput; - this.ords = IntBlock.newBlockBuilder(emitBatchSize); + this.ords = blockFactory.newIntBlockBuilder(emitBatchSize); } protected final void addedValue(int position) { @@ -174,13 +171,20 @@ protected final void addedValueInMultivaluePosition(int position) { } protected final void emitOrds() { - addInput.add(positionOffset, ords.build()); + try (IntBlock ordsBlock = ords.build()) { + addInput.add(positionOffset, ordsBlock); + } } private void rollover(int position) { emitOrds(); positionOffset = position; - ords = IntBlock.newBlockBuilder(emitBatchSize); // TODO add a clear method to the builder? + ords = blockFactory.newIntBlockBuilder(emitBatchSize); // TODO add a clear method to the builder? + } + + @Override + public final void close() { + ords.close(); } } @@ -197,13 +201,24 @@ static int add(long[] seen, int nextSeen, long v) { @Override public Block[] getKeys() { int positions = (int) hash.size(); - LongVector.Builder keys1 = blockFactory.newLongVectorBuilder(positions); - LongVector.Builder keys2 = blockFactory.newLongVectorBuilder(positions); - for (long i = 0; i < positions; i++) { - keys1.appendLong(hash.getKey1(i)); - keys2.appendLong(hash.getKey2(i)); + LongVector k1 = null; + LongVector k2 = null; + try ( + LongVector.Builder keys1 = blockFactory.newLongVectorBuilder(positions); + LongVector.Builder keys2 = blockFactory.newLongVectorBuilder(positions) + ) { + for (long i = 0; i < positions; i++) { + keys1.appendLong(hash.getKey1(i)); + keys2.appendLong(hash.getKey2(i)); + } + k1 = keys1.build(); + k2 = keys2.build(); + } finally { + if (k2 == null) { + Releasables.close(k1); + } } - return new Block[] { keys1.build().asBlock(), keys2.build().asBlock() }; + return new Block[] { k1.asBlock(), k2.asBlock() }; } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java index a624a3ec68100..06b833974a5db 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java @@ -73,7 +73,9 @@ public void add(Page page, GroupingAggregatorFunction.AddInput addInput) { } void add(Page page, GroupingAggregatorFunction.AddInput addInput, int batchSize) { - new AddWork(page, addInput, batchSize).add(); + try (AddWork work = new AddWork(page, addInput, batchSize)) { + work.add(); + } } private static class Group { @@ -95,7 +97,7 @@ class AddWork extends LongLongBlockHash.AbstractAddBlock { int position; AddWork(Page page, GroupingAggregatorFunction.AddInput addInput, int batchSize) { - super(emitBatchSize, addInput); + super(blockFactory, emitBatchSize, addInput); for (Group group : groups) { group.encoder = MultivalueDedupe.batchEncoder(new Block.Ref(page.getBlock(group.spec.channel()), page), batchSize, true); } @@ -193,45 +195,54 @@ public Block[] getKeys() { int size = Math.toIntExact(bytesRefHash.size()); BatchEncoder.Decoder[] decoders = new BatchEncoder.Decoder[groups.length]; Block.Builder[] builders = new Block.Builder[groups.length]; - for (int g = 0; g < builders.length; g++) { - ElementType elementType = groups[g].spec.elementType(); - decoders[g] = BatchEncoder.decoder(elementType); - builders[g] = elementType.newBlockBuilder(size, blockFactory); - } + try { + for (int g = 0; g < builders.length; g++) { + ElementType elementType = groups[g].spec.elementType(); + decoders[g] = BatchEncoder.decoder(elementType); + builders[g] = elementType.newBlockBuilder(size, blockFactory); + } - BytesRef[] values = new BytesRef[(int) Math.min(100, bytesRefHash.size())]; - BytesRef[] nulls = new BytesRef[values.length]; - for (int offset = 0; offset < values.length; offset++) { - values[offset] = new BytesRef(); - nulls[offset] = new BytesRef(); - nulls[offset].length = nullTrackingBytes; - } - int offset = 0; - for (int i = 0; i < bytesRefHash.size(); i++) { - values[offset] = bytesRefHash.get(i, values[offset]); + BytesRef[] values = new BytesRef[(int) Math.min(100, bytesRefHash.size())]; + BytesRef[] nulls = new BytesRef[values.length]; + for (int offset = 0; offset < values.length; offset++) { + values[offset] = new BytesRef(); + nulls[offset] = new BytesRef(); + nulls[offset].length = nullTrackingBytes; + } + int offset = 0; + for (int i = 0; i < bytesRefHash.size(); i++) { + values[offset] = bytesRefHash.get(i, values[offset]); - // Reference the null bytes in the nulls array and values in the values - nulls[offset].bytes = values[offset].bytes; - nulls[offset].offset = values[offset].offset; - values[offset].offset += nullTrackingBytes; - values[offset].length -= nullTrackingBytes; + // Reference the null bytes in the nulls array and values in the values + nulls[offset].bytes = values[offset].bytes; + nulls[offset].offset = values[offset].offset; + values[offset].offset += nullTrackingBytes; + values[offset].length -= nullTrackingBytes; - offset++; - if (offset == values.length) { + offset++; + if (offset == values.length) { + readKeys(decoders, builders, nulls, values, offset); + offset = 0; + } + } + if (offset > 0) { readKeys(decoders, builders, nulls, values, offset); - offset = 0; } - } - if (offset > 0) { - readKeys(decoders, builders, nulls, values, offset); - } - Block[] keyBlocks = new Block[groups.length]; - for (int g = 0; g < keyBlocks.length; g++) { - keyBlocks[g] = builders[g].build(); + Block[] keyBlocks = new Block[groups.length]; + try { + for (int g = 0; g < keyBlocks.length; g++) { + keyBlocks[g] = builders[g].build(); + } + } finally { + if (keyBlocks[keyBlocks.length - 1] == null) { + Releasables.closeExpectNoException(keyBlocks); + } + } + return keyBlocks; + } finally { + Releasables.closeExpectNoException(builders); } - Releasables.closeExpectNoException(builders); - return keyBlocks; } private void readKeys(BatchEncoder.Decoder[] decoders, Block.Builder[] builders, BytesRef[] nulls, BytesRef[] values, int count) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java index ad5dfbf298200..d27f872881460 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java @@ -109,7 +109,7 @@ public BooleanBlock.Builder newBooleanBlockBuilder(int estimatedSize) { return new BooleanBlockBuilder(estimatedSize, this); } - BooleanVector.FixedBuilder newBooleanVectorFixedBuilder(int size) { + public BooleanVector.FixedBuilder newBooleanVectorFixedBuilder(int size) { return new BooleanVectorFixedBuilder(size, this); } @@ -172,7 +172,7 @@ public IntVector.Builder newIntVectorBuilder(int estimatedSize) { return new IntVectorBuilder(estimatedSize, this); } - IntVector.FixedBuilder newIntVectorFixedBuilder(int size) { + public IntVector.FixedBuilder newIntVectorFixedBuilder(int size) { return new IntVectorFixedBuilder(size, this); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java index 4cd2c717eb57a..5dfba49b404e8 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashRandomizedTests.java @@ -11,22 +11,24 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.compute.data.BasicBlockTests; import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.MockBlockFactory; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.HashAggregationOperator; import org.elasticsearch.compute.operator.MultivalueDedupeTests; import org.elasticsearch.core.Releasables; +import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ListMatcher; -import org.junit.After; import java.util.ArrayList; import java.util.Comparator; @@ -45,11 +47,6 @@ //@TestLogging(value = "org.elasticsearch.compute:TRACE", reason = "debug") public class BlockHashRandomizedTests extends ESTestCase { - - final CircuitBreaker breaker = new MockBigArrays.LimitedBreaker("esql-test-breaker", ByteSizeValue.ofGb(1)); - final BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, mockBreakerService(breaker)); - final MockBlockFactory blockFactory = new MockBlockFactory(breaker, bigArrays); - @ParametersFactory public static List params() { List params = new ArrayList<>(); @@ -101,19 +98,33 @@ public BlockHashRandomizedTests( this.allowedTypes = allowedTypes; } - @After - public void checkBreaker() { - assertThat(breaker.getUsed(), is(0L)); + public void test() { + CircuitBreaker breaker = new MockBigArrays.LimitedBreaker("esql-test-breaker", ByteSizeValue.ofGb(1)); + BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, mockBreakerService(breaker)); + test(new MockBlockFactory(breaker, bigArrays)); } - public void test() { + public void testWithCranky() { + CircuitBreakerService service = new CrankyCircuitBreakerService(); + CircuitBreaker breaker = service.getBreaker(CircuitBreaker.REQUEST); + BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, service); + try { + test(new MockBlockFactory(breaker, bigArrays)); + logger.info("cranky let us finish!"); + } catch (CircuitBreakingException e) { + logger.info("cranky", e); + assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE)); + } + } + + private void test(MockBlockFactory blockFactory) { List types = randomList(groups, groups, () -> randomFrom(allowedTypes)); BasicBlockTests.RandomBlock[] randomBlocks = new BasicBlockTests.RandomBlock[types.size()]; Block[] blocks = new Block[types.size()]; int pageCount = between(1, 10); int positionCount = 100; int emitBatchSize = 100; - try (BlockHash blockHash = newBlockHash(emitBatchSize, types)) { + try (BlockHash blockHash = newBlockHash(blockFactory, emitBatchSize, types)) { /* * Only the long/long, long/bytes_ref, and bytes_ref/long implementations don't collect nulls. */ @@ -146,7 +157,6 @@ public void test() { assertThat(ordsAndKeys.ords().getTotalValueCount(), lessThanOrEqualTo(emitBatchSize)); } batchCount[0]++; - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty().asBlock()); }, blocks); if (usingSingle) { assertThat(batchCount[0], equalTo(1)); @@ -184,14 +194,15 @@ public void test() { blockFactory.ensureAllBlocksAreReleased(); } } + assertThat(blockFactory.breaker().getUsed(), is(0L)); } - private BlockHash newBlockHash(int emitBatchSize, List types) { + private BlockHash newBlockHash(BlockFactory blockFactory, int emitBatchSize, List types) { List specs = new ArrayList<>(types.size()); for (int c = 0; c < types.size(); c++) { specs.add(new HashAggregationOperator.GroupSpec(c, types.get(c))); } - DriverContext driverContext = new DriverContext(bigArrays, blockFactory); + DriverContext driverContext = new DriverContext(blockFactory.bigArrays(), blockFactory); return forcePackedHash ? new PackedValuesBlockHash(specs, driverContext, emitBatchSize) : BlockHash.build(specs, driverContext, emitBatchSize, true); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java index 86c7f64dc3816..567f58d0dee75 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java @@ -7,11 +7,11 @@ package org.elasticsearch.compute.aggregation.blockhash; +import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.inject.name.Named; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.MockBigArrays; @@ -25,6 +25,7 @@ import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.MockBlockFactory; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.HashAggregationOperator; @@ -55,7 +56,7 @@ public class BlockHashTests extends ESTestCase { final CircuitBreaker breaker = new MockBigArrays.LimitedBreaker("esql-test-breaker", ByteSizeValue.ofGb(1)); final BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, mockBreakerService(breaker)); - final BlockFactory blockFactory = BlockFactory.getInstance(breaker, bigArrays); + final MockBlockFactory blockFactory = new MockBlockFactory(breaker, bigArrays); @ParametersFactory public static List params() { @@ -67,603 +68,565 @@ public static List params() { @After public void checkBreaker() { + blockFactory.ensureAllBlocksAreReleased(); assertThat(breaker.getUsed(), is(0L)); } private final boolean forcePackedHash; - public BlockHashTests(@Named("forcePackedHash") boolean forcePackedHash) { + public BlockHashTests(@Name("forcePackedHash") boolean forcePackedHash) { this.forcePackedHash = forcePackedHash; } public void testIntHash() { int[] values = new int[] { 1, 2, 3, 1, 2, 3, 1, 2, 3 }; - IntBlock block = BlockFactory.getNonBreakingInstance().newIntArrayVector(values, values.length).asBlock(); - - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:INT], entries=3, size=")); - assertOrds(ordsAndKeys.ords, 0, 1, 2, 0, 1, 2, 0, 1, 2); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); - } else { - assertThat(ordsAndKeys.description, equalTo("IntBlockHash{channel=0, entries=3, seenNull=false}")); - assertOrds(ordsAndKeys.ords, 1, 2, 3, 1, 2, 3, 1, 2, 3); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 4))); - } - assertKeys(ordsAndKeys.keys, 1, 2, 3); - // we close these explicitly in the test. In common operation the operator is in charge of closing these. - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:INT], entries=3, size=")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 0, 1, 2, 0, 1, 2); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); + } else { + assertThat(ordsAndKeys.description, equalTo("IntBlockHash{channel=0, entries=3, seenNull=false}")); + assertOrds(ordsAndKeys.ords, 1, 2, 3, 1, 2, 3, 1, 2, 3); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 4))); + } + assertKeys(ordsAndKeys.keys, 1, 2, 3); + }, blockFactory.newIntArrayVector(values, values.length).asBlock()); } public void testIntHashWithNulls() { - IntBlock.Builder builder = BlockFactory.getNonBreakingInstance().newIntBlockBuilder(4); - builder.appendInt(0); - builder.appendNull(); - builder.appendInt(2); - builder.appendNull(); - - OrdsAndKeys ordsAndKeys = hash(builder.build()); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:INT], entries=3, size=")); - assertOrds(ordsAndKeys.ords, 0, 1, 2, 1); - assertKeys(ordsAndKeys.keys, 0, null, 2); - } else { - assertThat(ordsAndKeys.description, equalTo("IntBlockHash{channel=0, entries=2, seenNull=true}")); - assertOrds(ordsAndKeys.ords, 1, 0, 2, 0); - assertKeys(ordsAndKeys.keys, null, 0, 2); + try (IntBlock.Builder builder = blockFactory.newIntBlockBuilder(4)) { + builder.appendInt(0); + builder.appendNull(); + builder.appendInt(2); + builder.appendNull(); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:INT], entries=3, size=")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 1); + assertKeys(ordsAndKeys.keys, 0, null, 2); + } else { + assertThat(ordsAndKeys.description, equalTo("IntBlockHash{channel=0, entries=2, seenNull=true}")); + assertOrds(ordsAndKeys.ords, 1, 0, 2, 0); + assertKeys(ordsAndKeys.keys, null, 0, 2); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); + }, builder); } - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testIntHashWithMultiValuedFields() { - var builder = BlockFactory.getNonBreakingInstance().newIntBlockBuilder(8); - builder.appendInt(1); - builder.beginPositionEntry(); - builder.appendInt(1); - builder.appendInt(2); - builder.endPositionEntry(); - builder.beginPositionEntry(); - builder.appendInt(3); - builder.appendInt(1); - builder.endPositionEntry(); - builder.beginPositionEntry(); - builder.appendInt(3); - builder.appendInt(3); - builder.endPositionEntry(); - builder.appendNull(); - builder.beginPositionEntry(); - builder.appendInt(3); - builder.appendInt(2); - builder.appendInt(1); - builder.endPositionEntry(); - - OrdsAndKeys ordsAndKeys = hash(builder.build()); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:INT], entries=4, size=")); - assertOrds( - ordsAndKeys.ords, - new int[] { 0 }, - new int[] { 0, 1 }, - new int[] { 2, 0 }, - new int[] { 2 }, - new int[] { 3 }, - new int[] { 2, 1, 0 } - ); - assertKeys(ordsAndKeys.keys, 1, 2, 3, null); - } else { - assertThat(ordsAndKeys.description, equalTo("IntBlockHash{channel=0, entries=3, seenNull=true}")); - assertOrds( - ordsAndKeys.ords, - new int[] { 1 }, - new int[] { 1, 2 }, - new int[] { 3, 1 }, - new int[] { 3 }, - new int[] { 0 }, - new int[] { 3, 2, 1 } - ); - assertKeys(ordsAndKeys.keys, null, 1, 2, 3); + try (IntBlock.Builder builder = blockFactory.newIntBlockBuilder(8)) { + builder.appendInt(1); + builder.beginPositionEntry(); + builder.appendInt(1); + builder.appendInt(2); + builder.endPositionEntry(); + builder.beginPositionEntry(); + builder.appendInt(3); + builder.appendInt(1); + builder.endPositionEntry(); + builder.beginPositionEntry(); + builder.appendInt(3); + builder.appendInt(3); + builder.endPositionEntry(); + builder.appendNull(); + builder.beginPositionEntry(); + builder.appendInt(3); + builder.appendInt(2); + builder.appendInt(1); + builder.endPositionEntry(); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:INT], entries=4, size=")); + assertOrds( + ordsAndKeys.ords, + new int[] { 0 }, + new int[] { 0, 1 }, + new int[] { 2, 0 }, + new int[] { 2 }, + new int[] { 3 }, + new int[] { 2, 1, 0 } + ); + assertKeys(ordsAndKeys.keys, 1, 2, 3, null); + } else { + assertThat(ordsAndKeys.description, equalTo("IntBlockHash{channel=0, entries=3, seenNull=true}")); + assertOrds( + ordsAndKeys.ords, + new int[] { 1 }, + new int[] { 1, 2 }, + new int[] { 3, 1 }, + new int[] { 3 }, + new int[] { 0 }, + new int[] { 3, 2, 1 } + ); + assertKeys(ordsAndKeys.keys, null, 1, 2, 3); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); + }, builder); } - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testLongHash() { long[] values = new long[] { 2, 1, 4, 2, 4, 1, 3, 4 }; - LongBlock block = BlockFactory.getNonBreakingInstance().newLongArrayVector(values, values.length).asBlock(); - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG], entries=4, size=")); - assertOrds(ordsAndKeys.ords, 0, 1, 2, 0, 2, 1, 3, 2); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); - } else { - assertThat(ordsAndKeys.description, equalTo("LongBlockHash{channel=0, entries=4, seenNull=false}")); - assertOrds(ordsAndKeys.ords, 1, 2, 3, 1, 3, 2, 4, 3); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 5))); - } - assertKeys(ordsAndKeys.keys, 2L, 1L, 4L, 3L); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG], entries=4, size=")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 0, 2, 1, 3, 2); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); + } else { + assertThat(ordsAndKeys.description, equalTo("LongBlockHash{channel=0, entries=4, seenNull=false}")); + assertOrds(ordsAndKeys.ords, 1, 2, 3, 1, 3, 2, 4, 3); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 5))); + } + assertKeys(ordsAndKeys.keys, 2L, 1L, 4L, 3L); + }, blockFactory.newLongArrayVector(values, values.length).asBlock()); } public void testLongHashWithNulls() { - LongBlock.Builder builder = BlockFactory.getNonBreakingInstance().newLongBlockBuilder(4); - builder.appendLong(0); - builder.appendNull(); - builder.appendLong(2); - builder.appendNull(); - - OrdsAndKeys ordsAndKeys = hash(builder.build()); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG], entries=3, size=")); - assertOrds(ordsAndKeys.ords, 0, 1, 2, 1); - assertKeys(ordsAndKeys.keys, 0L, null, 2L); - } else { - assertThat(ordsAndKeys.description, equalTo("LongBlockHash{channel=0, entries=2, seenNull=true}")); - assertOrds(ordsAndKeys.ords, 1, 0, 2, 0); - assertKeys(ordsAndKeys.keys, null, 0L, 2L); + try (LongBlock.Builder builder = blockFactory.newLongBlockBuilder(4)) { + builder.appendLong(0); + builder.appendNull(); + builder.appendLong(2); + builder.appendNull(); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG], entries=3, size=")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 1); + assertKeys(ordsAndKeys.keys, 0L, null, 2L); + } else { + assertThat(ordsAndKeys.description, equalTo("LongBlockHash{channel=0, entries=2, seenNull=true}")); + assertOrds(ordsAndKeys.ords, 1, 0, 2, 0); + assertKeys(ordsAndKeys.keys, null, 0L, 2L); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); + }, builder); } - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testLongHashWithMultiValuedFields() { - var builder = BlockFactory.getNonBreakingInstance().newLongBlockBuilder(8); - builder.appendLong(1); - builder.beginPositionEntry(); - builder.appendLong(1); - builder.appendLong(2); - builder.appendLong(3); - builder.endPositionEntry(); - builder.beginPositionEntry(); - builder.appendLong(1); - builder.appendLong(1); - builder.endPositionEntry(); - builder.beginPositionEntry(); - builder.appendLong(3); - builder.endPositionEntry(); - builder.appendNull(); - builder.beginPositionEntry(); - builder.appendLong(3); - builder.appendLong(2); - builder.appendLong(1); - builder.endPositionEntry(); - - Block block = builder.build(); - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG], entries=4, size=")); - assertOrds( - ordsAndKeys.ords, - new int[] { 0 }, - new int[] { 0, 1, 2 }, - new int[] { 0 }, - new int[] { 2 }, - new int[] { 3 }, - new int[] { 2, 1, 0 } - ); - assertKeys(ordsAndKeys.keys, 1L, 2L, 3L, null); - } else { - assertThat(ordsAndKeys.description, equalTo("LongBlockHash{channel=0, entries=3, seenNull=true}")); - assertOrds( - ordsAndKeys.ords, - new int[] { 1 }, - new int[] { 1, 2, 3 }, - new int[] { 1 }, - new int[] { 3 }, - new int[] { 0 }, - new int[] { 3, 2, 1 } - ); - assertKeys(ordsAndKeys.keys, null, 1L, 2L, 3L); + try (LongBlock.Builder builder = blockFactory.newLongBlockBuilder(8)) { + builder.appendLong(1); + builder.beginPositionEntry(); + builder.appendLong(1); + builder.appendLong(2); + builder.appendLong(3); + builder.endPositionEntry(); + builder.beginPositionEntry(); + builder.appendLong(1); + builder.appendLong(1); + builder.endPositionEntry(); + builder.beginPositionEntry(); + builder.appendLong(3); + builder.endPositionEntry(); + builder.appendNull(); + builder.beginPositionEntry(); + builder.appendLong(3); + builder.appendLong(2); + builder.appendLong(1); + builder.endPositionEntry(); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG], entries=4, size=")); + assertOrds( + ordsAndKeys.ords, + new int[] { 0 }, + new int[] { 0, 1, 2 }, + new int[] { 0 }, + new int[] { 2 }, + new int[] { 3 }, + new int[] { 2, 1, 0 } + ); + assertKeys(ordsAndKeys.keys, 1L, 2L, 3L, null); + } else { + assertThat(ordsAndKeys.description, equalTo("LongBlockHash{channel=0, entries=3, seenNull=true}")); + assertOrds( + ordsAndKeys.ords, + new int[] { 1 }, + new int[] { 1, 2, 3 }, + new int[] { 1 }, + new int[] { 3 }, + new int[] { 0 }, + new int[] { 3, 2, 1 } + ); + assertKeys(ordsAndKeys.keys, null, 1L, 2L, 3L); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); + }, builder); } - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testDoubleHash() { double[] values = new double[] { 2.0, 1.0, 4.0, 2.0, 4.0, 1.0, 3.0, 4.0 }; - DoubleBlock block = BlockFactory.getNonBreakingInstance().newDoubleArrayVector(values, values.length).asBlock(); - OrdsAndKeys ordsAndKeys = hash(block); - - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:DOUBLE], entries=4, size=")); - assertOrds(ordsAndKeys.ords, 0, 1, 2, 0, 2, 1, 3, 2); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); - } else { - assertThat(ordsAndKeys.description, equalTo("DoubleBlockHash{channel=0, entries=4, seenNull=false}")); - assertOrds(ordsAndKeys.ords, 1, 2, 3, 1, 3, 2, 4, 3); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 5))); - } - assertKeys(ordsAndKeys.keys, 2.0, 1.0, 4.0, 3.0); - // we close these explicitly in the test. In common operation the operator is in charge of closing these. - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:DOUBLE], entries=4, size=")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 0, 2, 1, 3, 2); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); + } else { + assertThat(ordsAndKeys.description, equalTo("DoubleBlockHash{channel=0, entries=4, seenNull=false}")); + assertOrds(ordsAndKeys.ords, 1, 2, 3, 1, 3, 2, 4, 3); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 5))); + } + assertKeys(ordsAndKeys.keys, 2.0, 1.0, 4.0, 3.0); + }, blockFactory.newDoubleArrayVector(values, values.length).asBlock()); } public void testDoubleHashWithNulls() { - DoubleBlock.Builder builder = BlockFactory.getNonBreakingInstance().newDoubleBlockBuilder(4); - builder.appendDouble(0); - builder.appendNull(); - builder.appendDouble(2); - builder.appendNull(); - - Block block = builder.build(); - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:DOUBLE], entries=3, size=")); - assertOrds(ordsAndKeys.ords, 0, 1, 2, 1); - assertKeys(ordsAndKeys.keys, 0.0, null, 2.0); - } else { - assertThat(ordsAndKeys.description, equalTo("DoubleBlockHash{channel=0, entries=2, seenNull=true}")); - assertOrds(ordsAndKeys.ords, 1, 0, 2, 0); - assertKeys(ordsAndKeys.keys, null, 0.0, 2.0); + try (DoubleBlock.Builder builder = blockFactory.newDoubleBlockBuilder(4)) { + builder.appendDouble(0); + builder.appendNull(); + builder.appendDouble(2); + builder.appendNull(); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:DOUBLE], entries=3, size=")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 1); + assertKeys(ordsAndKeys.keys, 0.0, null, 2.0); + } else { + assertThat(ordsAndKeys.description, equalTo("DoubleBlockHash{channel=0, entries=2, seenNull=true}")); + assertOrds(ordsAndKeys.ords, 1, 0, 2, 0); + assertKeys(ordsAndKeys.keys, null, 0.0, 2.0); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); + }, builder); } - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testDoubleHashWithMultiValuedFields() { - var builder = BlockFactory.getNonBreakingInstance().newDoubleBlockBuilder(8); - builder.appendDouble(1); - builder.beginPositionEntry(); - builder.appendDouble(2); - builder.appendDouble(3); - builder.endPositionEntry(); - builder.beginPositionEntry(); - builder.appendDouble(3); - builder.appendDouble(2); - builder.endPositionEntry(); - builder.beginPositionEntry(); - builder.appendDouble(1); - builder.endPositionEntry(); - builder.appendNull(); - builder.beginPositionEntry(); - builder.appendDouble(1); - builder.appendDouble(1); - builder.appendDouble(2); - builder.endPositionEntry(); - - Block block = builder.build(); - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:DOUBLE], entries=4, size=")); - assertOrds( - ordsAndKeys.ords, - new int[] { 0 }, - new int[] { 1, 2 }, - new int[] { 2, 1 }, - new int[] { 0 }, - new int[] { 3 }, - new int[] { 0, 1 } - ); - assertKeys(ordsAndKeys.keys, 1.0, 2.0, 3.0, null); - } else { - assertThat(ordsAndKeys.description, equalTo("DoubleBlockHash{channel=0, entries=3, seenNull=true}")); - assertOrds( - ordsAndKeys.ords, - new int[] { 1 }, - new int[] { 2, 3 }, - new int[] { 3, 2 }, - new int[] { 1 }, - new int[] { 0 }, - new int[] { 1, 2 } - ); - assertKeys(ordsAndKeys.keys, null, 1.0, 2.0, 3.0); + try (DoubleBlock.Builder builder = blockFactory.newDoubleBlockBuilder(8)) { + builder.appendDouble(1); + builder.beginPositionEntry(); + builder.appendDouble(2); + builder.appendDouble(3); + builder.endPositionEntry(); + builder.beginPositionEntry(); + builder.appendDouble(3); + builder.appendDouble(2); + builder.endPositionEntry(); + builder.beginPositionEntry(); + builder.appendDouble(1); + builder.endPositionEntry(); + builder.appendNull(); + builder.beginPositionEntry(); + builder.appendDouble(1); + builder.appendDouble(1); + builder.appendDouble(2); + builder.endPositionEntry(); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:DOUBLE], entries=4, size=")); + assertOrds( + ordsAndKeys.ords, + new int[] { 0 }, + new int[] { 1, 2 }, + new int[] { 2, 1 }, + new int[] { 0 }, + new int[] { 3 }, + new int[] { 0, 1 } + ); + assertKeys(ordsAndKeys.keys, 1.0, 2.0, 3.0, null); + } else { + assertThat(ordsAndKeys.description, equalTo("DoubleBlockHash{channel=0, entries=3, seenNull=true}")); + assertOrds( + ordsAndKeys.ords, + new int[] { 1 }, + new int[] { 2, 3 }, + new int[] { 3, 2 }, + new int[] { 1 }, + new int[] { 0 }, + new int[] { 1, 2 } + ); + assertKeys(ordsAndKeys.keys, null, 1.0, 2.0, 3.0); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); + }, builder); } - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testBasicBytesRefHash() { - var builder = BlockFactory.getNonBreakingInstance().newBytesRefBlockBuilder(8); - builder.appendBytesRef(new BytesRef("item-2")); - builder.appendBytesRef(new BytesRef("item-1")); - builder.appendBytesRef(new BytesRef("item-4")); - builder.appendBytesRef(new BytesRef("item-2")); - builder.appendBytesRef(new BytesRef("item-4")); - builder.appendBytesRef(new BytesRef("item-1")); - builder.appendBytesRef(new BytesRef("item-3")); - builder.appendBytesRef(new BytesRef("item-4")); - - Block block = builder.build(); - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=4, size=")); - assertThat(ordsAndKeys.description, endsWith("b}")); - assertOrds(ordsAndKeys.ords, 0, 1, 2, 0, 2, 1, 3, 2); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); - } else { - assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=4, size=")); - assertThat(ordsAndKeys.description, endsWith("b, seenNull=false}")); - assertOrds(ordsAndKeys.ords, 1, 2, 3, 1, 3, 2, 4, 3); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 5))); + try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(8)) { + builder.appendBytesRef(new BytesRef("item-2")); + builder.appendBytesRef(new BytesRef("item-1")); + builder.appendBytesRef(new BytesRef("item-4")); + builder.appendBytesRef(new BytesRef("item-2")); + builder.appendBytesRef(new BytesRef("item-4")); + builder.appendBytesRef(new BytesRef("item-1")); + builder.appendBytesRef(new BytesRef("item-3")); + builder.appendBytesRef(new BytesRef("item-4")); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=4, size=")); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 0, 2, 1, 3, 2); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); + } else { + assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=4, size=")); + assertThat(ordsAndKeys.description, endsWith("b, seenNull=false}")); + assertOrds(ordsAndKeys.ords, 1, 2, 3, 1, 3, 2, 4, 3); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 5))); + } + assertKeys(ordsAndKeys.keys, "item-2", "item-1", "item-4", "item-3"); + }, builder); } - assertKeys(ordsAndKeys.keys, "item-2", "item-1", "item-4", "item-3"); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testBytesRefHashWithNulls() { - BytesRefBlock.Builder builder = BlockFactory.getNonBreakingInstance().newBytesRefBlockBuilder(4); - builder.appendBytesRef(new BytesRef("cat")); - builder.appendNull(); - builder.appendBytesRef(new BytesRef("dog")); - builder.appendNull(); - - Block block = builder.build(); - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=3, size=")); - assertThat(ordsAndKeys.description, endsWith("b}")); - assertOrds(ordsAndKeys.ords, 0, 1, 2, 1); - assertKeys(ordsAndKeys.keys, "cat", null, "dog"); - } else { - assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=2, size=")); - assertThat(ordsAndKeys.description, endsWith("b, seenNull=true}")); - assertOrds(ordsAndKeys.ords, 1, 0, 2, 0); - assertKeys(ordsAndKeys.keys, null, "cat", "dog"); + try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(4)) { + builder.appendBytesRef(new BytesRef("cat")); + builder.appendNull(); + builder.appendBytesRef(new BytesRef("dog")); + builder.appendNull(); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=3, size=")); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 1); + assertKeys(ordsAndKeys.keys, "cat", null, "dog"); + } else { + assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=2, size=")); + assertThat(ordsAndKeys.description, endsWith("b, seenNull=true}")); + assertOrds(ordsAndKeys.ords, 1, 0, 2, 0); + assertKeys(ordsAndKeys.keys, null, "cat", "dog"); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); + }, builder); } - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testBytesRefHashWithMultiValuedFields() { - var builder = BlockFactory.getNonBreakingInstance().newBytesRefBlockBuilder(8); - builder.appendBytesRef(new BytesRef("foo")); - builder.beginPositionEntry(); - builder.appendBytesRef(new BytesRef("foo")); - builder.appendBytesRef(new BytesRef("bar")); - builder.endPositionEntry(); - builder.beginPositionEntry(); - builder.appendBytesRef(new BytesRef("bar")); - builder.appendBytesRef(new BytesRef("bort")); - builder.endPositionEntry(); - builder.beginPositionEntry(); - builder.appendBytesRef(new BytesRef("bort")); - builder.appendBytesRef(new BytesRef("bar")); - builder.endPositionEntry(); - builder.appendNull(); - builder.beginPositionEntry(); - builder.appendBytesRef(new BytesRef("bort")); - builder.appendBytesRef(new BytesRef("bort")); - builder.appendBytesRef(new BytesRef("bar")); - builder.endPositionEntry(); - - Block block = builder.build(); - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=4, size=")); - assertThat(ordsAndKeys.description, endsWith("b}")); - assertOrds( - ordsAndKeys.ords, - new int[] { 0 }, - new int[] { 0, 1 }, - new int[] { 1, 2 }, - new int[] { 2, 1 }, - new int[] { 3 }, - new int[] { 2, 1 } - ); - assertKeys(ordsAndKeys.keys, "foo", "bar", "bort", null); - } else { - assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=3, size=")); - assertThat(ordsAndKeys.description, endsWith("b, seenNull=true}")); - assertOrds( - ordsAndKeys.ords, - new int[] { 1 }, - new int[] { 1, 2 }, - new int[] { 2, 3 }, - new int[] { 3, 2 }, - new int[] { 0 }, - new int[] { 3, 2 } - ); - assertKeys(ordsAndKeys.keys, null, "foo", "bar", "bort"); + try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(8)) { + builder.appendBytesRef(new BytesRef("foo")); + builder.beginPositionEntry(); + builder.appendBytesRef(new BytesRef("foo")); + builder.appendBytesRef(new BytesRef("bar")); + builder.endPositionEntry(); + builder.beginPositionEntry(); + builder.appendBytesRef(new BytesRef("bar")); + builder.appendBytesRef(new BytesRef("bort")); + builder.endPositionEntry(); + builder.beginPositionEntry(); + builder.appendBytesRef(new BytesRef("bort")); + builder.appendBytesRef(new BytesRef("bar")); + builder.endPositionEntry(); + builder.appendNull(); + builder.beginPositionEntry(); + builder.appendBytesRef(new BytesRef("bort")); + builder.appendBytesRef(new BytesRef("bort")); + builder.appendBytesRef(new BytesRef("bar")); + builder.endPositionEntry(); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BYTES_REF], entries=4, size=")); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds( + ordsAndKeys.ords, + new int[] { 0 }, + new int[] { 0, 1 }, + new int[] { 1, 2 }, + new int[] { 2, 1 }, + new int[] { 3 }, + new int[] { 2, 1 } + ); + assertKeys(ordsAndKeys.keys, "foo", "bar", "bort", null); + } else { + assertThat(ordsAndKeys.description, startsWith("BytesRefBlockHash{channel=0, entries=3, size=")); + assertThat(ordsAndKeys.description, endsWith("b, seenNull=true}")); + assertOrds( + ordsAndKeys.ords, + new int[] { 1 }, + new int[] { 1, 2 }, + new int[] { 2, 3 }, + new int[] { 3, 2 }, + new int[] { 0 }, + new int[] { 3, 2 } + ); + assertKeys(ordsAndKeys.keys, null, "foo", "bar", "bort"); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); + }, builder); } - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testBooleanHashFalseFirst() { boolean[] values = new boolean[] { false, true, true, true, true }; - BooleanBlock block = BlockFactory.getNonBreakingInstance().newBooleanArrayVector(values, values.length).asBlock(); - - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=2, size=")); - assertOrds(ordsAndKeys.ords, 0, 1, 1, 1, 1); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 2))); - } else { - assertThat(ordsAndKeys.description, equalTo("BooleanBlockHash{channel=0, seenFalse=true, seenTrue=true, seenNull=false}")); - assertOrds(ordsAndKeys.ords, 1, 2, 2, 2, 2); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 3))); - } - assertKeys(ordsAndKeys.keys, false, true); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=2, size=")); + assertOrds(ordsAndKeys.ords, 0, 1, 1, 1, 1); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 2))); + } else { + assertThat(ordsAndKeys.description, equalTo("BooleanBlockHash{channel=0, seenFalse=true, seenTrue=true, seenNull=false}")); + assertOrds(ordsAndKeys.ords, 1, 2, 2, 2, 2); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 3))); + } + assertKeys(ordsAndKeys.keys, false, true); + }, blockFactory.newBooleanArrayVector(values, values.length).asBlock()); } public void testBooleanHashTrueFirst() { boolean[] values = new boolean[] { true, false, false, true, true }; - BooleanBlock block = BlockFactory.getNonBreakingInstance().newBooleanArrayVector(values, values.length).asBlock(); - - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=2, size=")); - assertOrds(ordsAndKeys.ords, 0, 1, 1, 0, 0); - assertKeys(ordsAndKeys.keys, true, false); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 2))); - } else { - assertThat(ordsAndKeys.description, equalTo("BooleanBlockHash{channel=0, seenFalse=true, seenTrue=true, seenNull=false}")); - assertOrds(ordsAndKeys.ords, 2, 1, 1, 2, 2); - assertKeys(ordsAndKeys.keys, false, true); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 3))); - } - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=2, size=")); + assertOrds(ordsAndKeys.ords, 0, 1, 1, 0, 0); + assertKeys(ordsAndKeys.keys, true, false); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 2))); + } else { + assertThat(ordsAndKeys.description, equalTo("BooleanBlockHash{channel=0, seenFalse=true, seenTrue=true, seenNull=false}")); + assertOrds(ordsAndKeys.ords, 2, 1, 1, 2, 2); + assertKeys(ordsAndKeys.keys, false, true); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(1, 3))); + } + }, blockFactory.newBooleanArrayVector(values, values.length).asBlock()); } public void testBooleanHashTrueOnly() { boolean[] values = new boolean[] { true, true, true, true }; - BooleanBlock block = BlockFactory.getNonBreakingInstance().newBooleanArrayVector(values, values.length).asBlock(); - - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=1, size=")); - assertOrds(ordsAndKeys.ords, 0, 0, 0, 0); - assertKeys(ordsAndKeys.keys, true); - assertThat(ordsAndKeys.nonEmpty, equalTo(IntVector.newVectorBuilder(1).appendInt(0).build())); - } else { - assertThat(ordsAndKeys.description, equalTo("BooleanBlockHash{channel=0, seenFalse=false, seenTrue=true, seenNull=false}")); - assertOrds(ordsAndKeys.ords, 2, 2, 2, 2); - assertKeys(ordsAndKeys.keys, true); - assertThat(ordsAndKeys.nonEmpty, equalTo(IntVector.newVectorBuilder(1).appendInt(2).build())); - } - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=1, size=")); + assertOrds(ordsAndKeys.ords, 0, 0, 0, 0); + assertKeys(ordsAndKeys.keys, true); + assertThat(ordsAndKeys.nonEmpty, equalTo(IntVector.newVectorBuilder(1).appendInt(0).build())); + } else { + assertThat(ordsAndKeys.description, equalTo("BooleanBlockHash{channel=0, seenFalse=false, seenTrue=true, seenNull=false}")); + assertOrds(ordsAndKeys.ords, 2, 2, 2, 2); + assertKeys(ordsAndKeys.keys, true); + assertThat(ordsAndKeys.nonEmpty, equalTo(IntVector.newVectorBuilder(1).appendInt(2).build())); + } + }, blockFactory.newBooleanArrayVector(values, values.length).asBlock()); } public void testBooleanHashFalseOnly() { boolean[] values = new boolean[] { false, false, false, false }; - BooleanBlock block = BlockFactory.getNonBreakingInstance().newBooleanArrayVector(values, values.length).asBlock(); - - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=1, size=")); - assertOrds(ordsAndKeys.ords, 0, 0, 0, 0); - assertThat(ordsAndKeys.nonEmpty, equalTo(IntVector.newVectorBuilder(1).appendInt(0).build())); - } else { - assertThat(ordsAndKeys.description, equalTo("BooleanBlockHash{channel=0, seenFalse=true, seenTrue=false, seenNull=false}")); - assertOrds(ordsAndKeys.ords, 1, 1, 1, 1); - assertThat(ordsAndKeys.nonEmpty, equalTo(IntVector.newVectorBuilder(1).appendInt(1).build())); - } - assertKeys(ordsAndKeys.keys, false); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=1, size=")); + assertOrds(ordsAndKeys.ords, 0, 0, 0, 0); + assertThat(ordsAndKeys.nonEmpty, equalTo(IntVector.newVectorBuilder(1).appendInt(0).build())); + } else { + assertThat(ordsAndKeys.description, equalTo("BooleanBlockHash{channel=0, seenFalse=true, seenTrue=false, seenNull=false}")); + assertOrds(ordsAndKeys.ords, 1, 1, 1, 1); + assertThat(ordsAndKeys.nonEmpty, equalTo(IntVector.newVectorBuilder(1).appendInt(1).build())); + } + assertKeys(ordsAndKeys.keys, false); + }, blockFactory.newBooleanArrayVector(values, values.length).asBlock()); } public void testBooleanHashWithNulls() { - BooleanBlock.Builder builder = BlockFactory.getNonBreakingInstance().newBooleanBlockBuilder(4); - builder.appendBoolean(false); - builder.appendNull(); - builder.appendBoolean(true); - builder.appendNull(); - - Block block = builder.build(); - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=3, size=")); - assertOrds(ordsAndKeys.ords, 0, 1, 2, 1); - assertKeys(ordsAndKeys.keys, false, null, true); - } else { - assertThat(ordsAndKeys.description, equalTo("BooleanBlockHash{channel=0, seenFalse=true, seenTrue=true, seenNull=true}")); - assertOrds(ordsAndKeys.ords, 1, 0, 2, 0); - assertKeys(ordsAndKeys.keys, null, false, true); + try (BooleanBlock.Builder builder = blockFactory.newBooleanBlockBuilder(4)) { + builder.appendBoolean(false); + builder.appendNull(); + builder.appendBoolean(true); + builder.appendNull(); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=3, size=")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 1); + assertKeys(ordsAndKeys.keys, false, null, true); + } else { + assertThat( + ordsAndKeys.description, + equalTo("BooleanBlockHash{channel=0, seenFalse=true, seenTrue=true, seenNull=true}") + ); + assertOrds(ordsAndKeys.ords, 1, 0, 2, 0); + assertKeys(ordsAndKeys.keys, null, false, true); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); + }, builder); } - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testBooleanHashWithMultiValuedFields() { - var builder = BlockFactory.getNonBreakingInstance().newBooleanBlockBuilder(8); - builder.appendBoolean(false); - builder.beginPositionEntry(); - builder.appendBoolean(false); - builder.appendBoolean(true); - builder.endPositionEntry(); - builder.beginPositionEntry(); - builder.appendBoolean(true); - builder.appendBoolean(false); - builder.endPositionEntry(); - builder.beginPositionEntry(); - builder.appendBoolean(true); - builder.endPositionEntry(); - builder.appendNull(); - builder.beginPositionEntry(); - builder.appendBoolean(true); - builder.appendBoolean(true); - builder.appendBoolean(false); - builder.endPositionEntry(); - - Block block = builder.build(); - OrdsAndKeys ordsAndKeys = hash(block); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=3, size=")); - assertOrds( - ordsAndKeys.ords, - new int[] { 0 }, - new int[] { 0, 1 }, - new int[] { 0, 1 }, // Order is not preserved - new int[] { 1 }, - new int[] { 2 }, - new int[] { 0, 1 } - ); - assertKeys(ordsAndKeys.keys, false, true, null); - } else { - assertThat(ordsAndKeys.description, equalTo("BooleanBlockHash{channel=0, seenFalse=true, seenTrue=true, seenNull=true}")); - assertOrds( - ordsAndKeys.ords, - new int[] { 1 }, - new int[] { 1, 2 }, - new int[] { 1, 2 }, // Order is not preserved - new int[] { 2 }, - new int[] { 0 }, - new int[] { 1, 2 } - ); - assertKeys(ordsAndKeys.keys, null, false, true); + try (BooleanBlock.Builder builder = blockFactory.newBooleanBlockBuilder(8)) { + builder.appendBoolean(false); + builder.beginPositionEntry(); + builder.appendBoolean(false); + builder.appendBoolean(true); + builder.endPositionEntry(); + builder.beginPositionEntry(); + builder.appendBoolean(true); + builder.appendBoolean(false); + builder.endPositionEntry(); + builder.beginPositionEntry(); + builder.appendBoolean(true); + builder.endPositionEntry(); + builder.appendNull(); + builder.beginPositionEntry(); + builder.appendBoolean(true); + builder.appendBoolean(true); + builder.appendBoolean(false); + builder.endPositionEntry(); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:BOOLEAN], entries=3, size=")); + assertOrds( + ordsAndKeys.ords, + new int[] { 0 }, + new int[] { 0, 1 }, + new int[] { 0, 1 }, // Order is not preserved + new int[] { 1 }, + new int[] { 2 }, + new int[] { 0, 1 } + ); + assertKeys(ordsAndKeys.keys, false, true, null); + } else { + assertThat( + ordsAndKeys.description, + equalTo("BooleanBlockHash{channel=0, seenFalse=true, seenTrue=true, seenNull=true}") + ); + assertOrds( + ordsAndKeys.ords, + new int[] { 1 }, + new int[] { 1, 2 }, + new int[] { 1, 2 }, // Order is not preserved + new int[] { 2 }, + new int[] { 0 }, + new int[] { 1, 2 } + ); + assertKeys(ordsAndKeys.keys, null, false, true); + } + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); + }, builder); } - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 3))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testLongLongHash() { long[] values1 = new long[] { 0, 1, 0, 1, 0, 1 }; - LongBlock block1 = BlockFactory.getNonBreakingInstance().newLongArrayVector(values1, values1.length).asBlock(); long[] values2 = new long[] { 0, 0, 0, 1, 1, 1 }; - LongBlock block2 = BlockFactory.getNonBreakingInstance().newLongArrayVector(values2, values2.length).asBlock(); - Object[][] expectedKeys = { new Object[] { 0L, 0L }, new Object[] { 1L, 0L }, new Object[] { 1L, 1L }, new Object[] { 0L, 1L } }; - - OrdsAndKeys ordsAndKeys = hash(block1, block2); - assertThat( - ordsAndKeys.description, - forcePackedHash - ? startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:LONG], entries=4, size=") - : equalTo("LongLongBlockHash{channels=[0,1], entries=4}") + hash(ordsAndKeys -> { + Object[][] expectedKeys = { + new Object[] { 0L, 0L }, + new Object[] { 1L, 0L }, + new Object[] { 1L, 1L }, + new Object[] { 0L, 1L } }; + + assertThat( + ordsAndKeys.description, + forcePackedHash + ? startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:LONG], entries=4, size=") + : equalTo("LongLongBlockHash{channels=[0,1], entries=4}") + ); + assertOrds(ordsAndKeys.ords, 0, 1, 0, 2, 3, 2); + assertKeys(ordsAndKeys.keys, expectedKeys); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); + }, + blockFactory.newLongArrayVector(values1, values1.length).asBlock(), + blockFactory.newLongArrayVector(values2, values2.length).asBlock() ); - assertOrds(ordsAndKeys.ords, 0, 1, 0, 2, 3, 2); - assertKeys(ordsAndKeys.keys, expectedKeys); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } private void append(LongBlock.Builder b1, LongBlock.Builder b2, long[] v1, long[] v2) { @@ -692,289 +655,282 @@ private void append(LongBlock.Builder b1, LongBlock.Builder b2, long[] v1, long[ } public void testLongLongHashWithMultiValuedFields() { - var b1 = BlockFactory.getNonBreakingInstance().newLongBlockBuilder(8); - var b2 = BlockFactory.getNonBreakingInstance().newLongBlockBuilder(8); - append(b1, b2, new long[] { 1, 2 }, new long[] { 10, 20 }); - append(b1, b2, new long[] { 1, 2 }, new long[] { 10 }); - append(b1, b2, new long[] { 1 }, new long[] { 10, 20 }); - append(b1, b2, new long[] { 1 }, new long[] { 10 }); - append(b1, b2, null, new long[] { 10 }); - append(b1, b2, new long[] { 1 }, null); - append(b1, b2, new long[] { 1, 1, 1 }, new long[] { 10, 10, 10 }); - append(b1, b2, new long[] { 1, 1, 2, 2 }, new long[] { 10, 20, 20 }); - append(b1, b2, new long[] { 1, 2, 3 }, new long[] { 30, 30, 10 }); - - Block block1 = b1.build(); - Block block2 = b2.build(); - OrdsAndKeys ordsAndKeys = hash(block1, block2); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:LONG], entries=10, size=")); - assertOrds( - ordsAndKeys.ords, - new int[] { 0, 1, 2, 3 }, - new int[] { 0, 2 }, - new int[] { 0, 1 }, - new int[] { 0 }, - new int[] { 4 }, - new int[] { 5 }, - new int[] { 0 }, - new int[] { 0, 1, 2, 3 }, - new int[] { 6, 0, 7, 2, 8, 9 } - ); - assertKeys( - ordsAndKeys.keys, - new Object[][] { - new Object[] { 1L, 10L }, - new Object[] { 1L, 20L }, - new Object[] { 2L, 10L }, - new Object[] { 2L, 20L }, - new Object[] { null, 10L }, - new Object[] { 1L, null }, - new Object[] { 1L, 30L }, - new Object[] { 2L, 30L }, - new Object[] { 3L, 30L }, - new Object[] { 3L, 10L }, } - ); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 10))); - } else { - assertThat(ordsAndKeys.description, equalTo("LongLongBlockHash{channels=[0,1], entries=8}")); - assertOrds( - ordsAndKeys.ords, - new int[] { 0, 1, 2, 3 }, - new int[] { 0, 2 }, - new int[] { 0, 1 }, - new int[] { 0 }, - null, - null, - new int[] { 0 }, - new int[] { 0, 1, 2, 3 }, - new int[] { 4, 0, 5, 2, 6, 7 } - ); - assertKeys( - ordsAndKeys.keys, - new Object[][] { - new Object[] { 1L, 10L }, - new Object[] { 1L, 20L }, - new Object[] { 2L, 10L }, - new Object[] { 2L, 20L }, - new Object[] { 1L, 30L }, - new Object[] { 2L, 30L }, - new Object[] { 3L, 30L }, - new Object[] { 3L, 10L }, } - ); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 8))); + try (LongBlock.Builder b1 = blockFactory.newLongBlockBuilder(8); LongBlock.Builder b2 = blockFactory.newLongBlockBuilder(8)) { + append(b1, b2, new long[] { 1, 2 }, new long[] { 10, 20 }); + append(b1, b2, new long[] { 1, 2 }, new long[] { 10 }); + append(b1, b2, new long[] { 1 }, new long[] { 10, 20 }); + append(b1, b2, new long[] { 1 }, new long[] { 10 }); + append(b1, b2, null, new long[] { 10 }); + append(b1, b2, new long[] { 1 }, null); + append(b1, b2, new long[] { 1, 1, 1 }, new long[] { 10, 10, 10 }); + append(b1, b2, new long[] { 1, 1, 2, 2 }, new long[] { 10, 20, 20 }); + append(b1, b2, new long[] { 1, 2, 3 }, new long[] { 30, 30, 10 }); + + hash(ordsAndKeys -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:LONG], entries=10, size=")); + assertOrds( + ordsAndKeys.ords, + new int[] { 0, 1, 2, 3 }, + new int[] { 0, 2 }, + new int[] { 0, 1 }, + new int[] { 0 }, + new int[] { 4 }, + new int[] { 5 }, + new int[] { 0 }, + new int[] { 0, 1, 2, 3 }, + new int[] { 6, 0, 7, 2, 8, 9 } + ); + assertKeys( + ordsAndKeys.keys, + new Object[][] { + new Object[] { 1L, 10L }, + new Object[] { 1L, 20L }, + new Object[] { 2L, 10L }, + new Object[] { 2L, 20L }, + new Object[] { null, 10L }, + new Object[] { 1L, null }, + new Object[] { 1L, 30L }, + new Object[] { 2L, 30L }, + new Object[] { 3L, 30L }, + new Object[] { 3L, 10L }, } + ); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 10))); + } else { + assertThat(ordsAndKeys.description, equalTo("LongLongBlockHash{channels=[0,1], entries=8}")); + assertOrds( + ordsAndKeys.ords, + new int[] { 0, 1, 2, 3 }, + new int[] { 0, 2 }, + new int[] { 0, 1 }, + new int[] { 0 }, + null, + null, + new int[] { 0 }, + new int[] { 0, 1, 2, 3 }, + new int[] { 4, 0, 5, 2, 6, 7 } + ); + assertKeys( + ordsAndKeys.keys, + new Object[][] { + new Object[] { 1L, 10L }, + new Object[] { 1L, 20L }, + new Object[] { 2L, 10L }, + new Object[] { 2L, 20L }, + new Object[] { 1L, 30L }, + new Object[] { 2L, 30L }, + new Object[] { 3L, 30L }, + new Object[] { 3L, 10L }, } + ); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 8))); + } + }, b1, b2); } - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testLongLongHashHugeCombinatorialExplosion() { long[] v1 = LongStream.range(0, 5000).toArray(); long[] v2 = LongStream.range(100, 200).toArray(); - var b1 = BlockFactory.getNonBreakingInstance().newLongBlockBuilder(v1.length); - var b2 = BlockFactory.getNonBreakingInstance().newLongBlockBuilder(v2.length); - append(b1, b2, v1, v2); - var block1 = b1.build(); - var block2 = b2.build(); - - int[] expectedEntries = new int[1]; - int pageSize = between(1000, 16 * 1024); - hash(ordsAndKeys -> { - int start = expectedEntries[0]; - expectedEntries[0] = Math.min(expectedEntries[0] + pageSize, v1.length * v2.length); - assertThat( - ordsAndKeys.description, - forcePackedHash - ? startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:LONG], entries=" + expectedEntries[0] + ", size=") - : equalTo("LongLongBlockHash{channels=[0,1], entries=" + expectedEntries[0] + "}") - ); - assertOrds(ordsAndKeys.ords, IntStream.range(start, expectedEntries[0]).toArray()); - assertKeys( - ordsAndKeys.keys, - IntStream.range(0, expectedEntries[0]) - .mapToObj(i -> new Object[] { v1[i / v2.length], v2[i % v2.length] }) - .toArray(l -> new Object[l][]) - ); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, expectedEntries[0]))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - }, pageSize, block1, block2); + try ( + LongBlock.Builder b1 = blockFactory.newLongBlockBuilder(v1.length); + LongBlock.Builder b2 = blockFactory.newLongBlockBuilder(v2.length) + ) { + append(b1, b2, v1, v2); + int[] expectedEntries = new int[1]; + int pageSize = between(1000, 16 * 1024); + hash(ordsAndKeys -> { + int start = expectedEntries[0]; + expectedEntries[0] = Math.min(expectedEntries[0] + pageSize, v1.length * v2.length); + assertThat( + ordsAndKeys.description, + forcePackedHash + ? startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:LONG], entries=" + expectedEntries[0] + ", size=") + : equalTo("LongLongBlockHash{channels=[0,1], entries=" + expectedEntries[0] + "}") + ); + assertOrds(ordsAndKeys.ords, IntStream.range(start, expectedEntries[0]).toArray()); + assertKeys( + ordsAndKeys.keys, + IntStream.range(0, expectedEntries[0]) + .mapToObj(i -> new Object[] { v1[i / v2.length], v2[i % v2.length] }) + .toArray(l -> new Object[l][]) + ); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, expectedEntries[0]))); + }, pageSize, b1, b2); - assertThat("misconfigured test", expectedEntries[0], greaterThan(0)); - assertThat(breaker.getUsed(), is(0L)); + assertThat("misconfigured test", expectedEntries[0], greaterThan(0)); + } } public void testIntLongHash() { int[] values1 = new int[] { 0, 1, 0, 1, 0, 1 }; - IntBlock block1 = BlockFactory.getNonBreakingInstance().newIntArrayVector(values1, values1.length).asBlock(); long[] values2 = new long[] { 0, 0, 0, 1, 1, 1 }; - LongBlock block2 = BlockFactory.getNonBreakingInstance().newLongArrayVector(values2, values2.length).asBlock(); Object[][] expectedKeys = { new Object[] { 0, 0L }, new Object[] { 1, 0L }, new Object[] { 1, 1L }, new Object[] { 0, 1L } }; - - OrdsAndKeys ordsAndKeys = hash(block1, block2); - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:INT, 1:LONG], entries=4, size=")); - assertThat(ordsAndKeys.description, endsWith("b}")); - assertOrds(ordsAndKeys.ords, 0, 1, 0, 2, 3, 2); - assertKeys(ordsAndKeys.keys, expectedKeys); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); + hash(ordsAndKeys -> { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:INT, 1:LONG], entries=4, size=")); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds(ordsAndKeys.ords, 0, 1, 0, 2, 3, 2); + assertKeys(ordsAndKeys.keys, expectedKeys); + }, + blockFactory.newIntArrayVector(values1, values1.length).asBlock(), + blockFactory.newLongArrayVector(values2, values2.length).asBlock() + ); } public void testLongDoubleHash() { long[] values1 = new long[] { 0, 1, 0, 1, 0, 1 }; - LongBlock block1 = BlockFactory.getNonBreakingInstance().newLongArrayVector(values1, values1.length).asBlock(); double[] values2 = new double[] { 0, 0, 0, 1, 1, 1 }; - DoubleBlock block2 = BlockFactory.getNonBreakingInstance().newDoubleArrayVector(values2, values2.length).asBlock(); Object[][] expectedKeys = { new Object[] { 0L, 0d }, new Object[] { 1L, 0d }, new Object[] { 1L, 1d }, new Object[] { 0L, 1d } }; - OrdsAndKeys ordsAndKeys = hash(block1, block2); - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:DOUBLE], entries=4, size=")); - assertThat(ordsAndKeys.description, endsWith("b}")); - assertOrds(ordsAndKeys.ords, 0, 1, 0, 2, 3, 2); - assertKeys(ordsAndKeys.keys, expectedKeys); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); + hash((OrdsAndKeys ordsAndKeys) -> { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:DOUBLE], entries=4, size=")); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds(ordsAndKeys.ords, 0, 1, 0, 2, 3, 2); + assertKeys(ordsAndKeys.keys, expectedKeys); + }, + blockFactory.newLongArrayVector(values1, values1.length).asBlock(), + blockFactory.newDoubleArrayVector(values2, values2.length).asBlock() + ); } public void testIntBooleanHash() { int[] values1 = new int[] { 0, 1, 0, 1, 0, 1 }; - IntBlock block1 = BlockFactory.getNonBreakingInstance().newIntArrayVector(values1, values1.length).asBlock(); boolean[] values2 = new boolean[] { false, false, false, true, true, true }; - BooleanBlock block2 = BlockFactory.getNonBreakingInstance().newBooleanArrayVector(values2, values2.length).asBlock(); Object[][] expectedKeys = { new Object[] { 0, false }, new Object[] { 1, false }, new Object[] { 1, true }, new Object[] { 0, true } }; - - OrdsAndKeys ordsAndKeys = hash(block1, block2); - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:INT, 1:BOOLEAN], entries=4, size=")); - assertThat(ordsAndKeys.description, endsWith("b}")); - assertOrds(ordsAndKeys.ords, 0, 1, 0, 2, 3, 2); - assertKeys(ordsAndKeys.keys, expectedKeys); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); + hash((OrdsAndKeys ordsAndKeys) -> { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:INT, 1:BOOLEAN], entries=4, size=")); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds(ordsAndKeys.ords, 0, 1, 0, 2, 3, 2); + assertKeys(ordsAndKeys.keys, expectedKeys); + }, + blockFactory.newIntArrayVector(values1, values1.length).asBlock(), + blockFactory.newBooleanArrayVector(values2, values2.length).asBlock() + ); } public void testLongLongHashWithNull() { - LongBlock.Builder b1 = BlockFactory.getNonBreakingInstance().newLongBlockBuilder(2); - LongBlock.Builder b2 = BlockFactory.getNonBreakingInstance().newLongBlockBuilder(2); - b1.appendLong(1); - b2.appendLong(0); - b1.appendNull(); - b2.appendNull(); - b1.appendLong(0); - b2.appendLong(1); - b1.appendLong(0); - b2.appendNull(); - b1.appendNull(); - b2.appendLong(0); - - OrdsAndKeys ordsAndKeys = hash(b1.build(), b2.build()); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:LONG], entries=5, size=")); - assertOrds(ordsAndKeys.ords, 0, 1, 2, 3, 4); - assertKeys( - ordsAndKeys.keys, - new Object[][] { - new Object[] { 1L, 0L }, - new Object[] { null, null }, - new Object[] { 0L, 1L }, - new Object[] { 0L, null }, - new Object[] { null, 0L }, } - ); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 5))); - } else { - assertThat(ordsAndKeys.description, equalTo("LongLongBlockHash{channels=[0,1], entries=2}")); - assertOrds(ordsAndKeys.ords, 0, null, 1, null, null); - assertKeys(ordsAndKeys.keys, new Object[][] { new Object[] { 1L, 0L }, new Object[] { 0L, 1L } }); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 2))); + try (LongBlock.Builder b1 = blockFactory.newLongBlockBuilder(2); LongBlock.Builder b2 = blockFactory.newLongBlockBuilder(2)) { + b1.appendLong(1); + b2.appendLong(0); + b1.appendNull(); + b2.appendNull(); + b1.appendLong(0); + b2.appendLong(1); + b1.appendLong(0); + b2.appendNull(); + b1.appendNull(); + b2.appendLong(0); + + hash((OrdsAndKeys ordsAndKeys) -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:LONG], entries=5, size=")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 3, 4); + assertKeys( + ordsAndKeys.keys, + new Object[][] { + new Object[] { 1L, 0L }, + new Object[] { null, null }, + new Object[] { 0L, 1L }, + new Object[] { 0L, null }, + new Object[] { null, 0L }, } + ); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 5))); + } else { + assertThat(ordsAndKeys.description, equalTo("LongLongBlockHash{channels=[0,1], entries=2}")); + assertOrds(ordsAndKeys.ords, 0, null, 1, null, null); + assertKeys(ordsAndKeys.keys, new Object[][] { new Object[] { 1L, 0L }, new Object[] { 0L, 1L } }); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 2))); + } + }, b1, b2); } - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); } public void testLongBytesRefHash() { - long[] values1 = new long[] { 0, 1, 0, 1, 0, 1 }; - LongBlock block1 = BlockFactory.getNonBreakingInstance().newLongArrayVector(values1, values1.length).asBlock(); - BytesRefBlock.Builder builder = BlockFactory.getNonBreakingInstance().newBytesRefBlockBuilder(8); - builder.appendBytesRef(new BytesRef("cat")); - builder.appendBytesRef(new BytesRef("cat")); - builder.appendBytesRef(new BytesRef("cat")); - builder.appendBytesRef(new BytesRef("dog")); - builder.appendBytesRef(new BytesRef("dog")); - builder.appendBytesRef(new BytesRef("dog")); - BytesRefBlock block2 = builder.build(); - Object[][] expectedKeys = { - new Object[] { 0L, "cat" }, - new Object[] { 1L, "cat" }, - new Object[] { 1L, "dog" }, - new Object[] { 0L, "dog" } }; - - OrdsAndKeys ordsAndKeys = hash(block1, block2); - assertThat( - ordsAndKeys.description, - startsWith( - forcePackedHash - ? "PackedValuesBlockHash{groups=[0:LONG, 1:BYTES_REF], entries=4, size=" - : "BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=4, size=" - ) - ); - assertThat(ordsAndKeys.description, endsWith("b}")); - assertOrds(ordsAndKeys.ords, 0, 1, 0, 2, 3, 2); - assertKeys(ordsAndKeys.keys, expectedKeys); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); + try ( + LongBlock.Builder b1 = blockFactory.newLongBlockBuilder(8); + BytesRefBlock.Builder b2 = blockFactory.newBytesRefBlockBuilder(8) + ) { + b1.appendLong(0); + b2.appendBytesRef(new BytesRef("cat")); + b1.appendLong(1); + b2.appendBytesRef(new BytesRef("cat")); + b1.appendLong(0); + b2.appendBytesRef(new BytesRef("cat")); + b1.appendLong(1); + b2.appendBytesRef(new BytesRef("dog")); + b1.appendLong(0); + b2.appendBytesRef(new BytesRef("dog")); + b1.appendLong(1); + b2.appendBytesRef(new BytesRef("dog")); + Object[][] expectedKeys = { + new Object[] { 0L, "cat" }, + new Object[] { 1L, "cat" }, + new Object[] { 1L, "dog" }, + new Object[] { 0L, "dog" } }; + + hash((OrdsAndKeys ordsAndKeys) -> { + assertThat( + ordsAndKeys.description, + startsWith( + forcePackedHash + ? "PackedValuesBlockHash{groups=[0:LONG, 1:BYTES_REF], entries=4, size=" + : "BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=4, size=" + ) + ); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds(ordsAndKeys.ords, 0, 1, 0, 2, 3, 2); + assertKeys(ordsAndKeys.keys, expectedKeys); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 4))); + }, b1, b2); + } } public void testLongBytesRefHashWithNull() { - LongBlock.Builder b1 = BlockFactory.getNonBreakingInstance().newLongBlockBuilder(2); - BytesRefBlock.Builder b2 = BlockFactory.getNonBreakingInstance().newBytesRefBlockBuilder(2); - b1.appendLong(1); - b2.appendBytesRef(new BytesRef("cat")); - b1.appendNull(); - b2.appendNull(); - b1.appendLong(0); - b2.appendBytesRef(new BytesRef("dog")); - b1.appendLong(0); - b2.appendNull(); - b1.appendNull(); - b2.appendBytesRef(new BytesRef("vanish")); - - OrdsAndKeys ordsAndKeys = hash(b1.build(), b2.build()); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:BYTES_REF], entries=5, size=")); - assertThat(ordsAndKeys.description, endsWith("b}")); - assertOrds(ordsAndKeys.ords, 0, 1, 2, 3, 4); - assertKeys( - ordsAndKeys.keys, - new Object[][] { - new Object[] { 1L, "cat" }, - new Object[] { null, null }, - new Object[] { 0L, "dog" }, - new Object[] { 1L, null }, - new Object[] { null, "vanish" } } - ); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 5))); - } else { - assertThat( - ordsAndKeys.description, - startsWith("BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=2, size=") - ); - assertThat(ordsAndKeys.description, endsWith("b}")); - assertOrds(ordsAndKeys.ords, 0, null, 1, null, null); - assertKeys(ordsAndKeys.keys, new Object[][] { new Object[] { 1L, "cat" }, new Object[] { 0L, "dog" } }); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 2))); + try ( + LongBlock.Builder b1 = blockFactory.newLongBlockBuilder(2); + BytesRefBlock.Builder b2 = blockFactory.newBytesRefBlockBuilder(2) + ) { + b1.appendLong(1); + b2.appendBytesRef(new BytesRef("cat")); + b1.appendNull(); + b2.appendNull(); + b1.appendLong(0); + b2.appendBytesRef(new BytesRef("dog")); + b1.appendLong(0); + b2.appendNull(); + b1.appendNull(); + b2.appendBytesRef(new BytesRef("vanish")); + + hash((OrdsAndKeys ordsAndKeys) -> { + if (forcePackedHash) { + assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:BYTES_REF], entries=5, size=")); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds(ordsAndKeys.ords, 0, 1, 2, 3, 4); + assertKeys( + ordsAndKeys.keys, + new Object[][] { + new Object[] { 1L, "cat" }, + new Object[] { null, null }, + new Object[] { 0L, "dog" }, + new Object[] { 1L, null }, + new Object[] { null, "vanish" } } + ); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 5))); + } else { + assertThat( + ordsAndKeys.description, + startsWith("BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=2, size=") + ); + assertThat(ordsAndKeys.description, endsWith("b}")); + assertOrds(ordsAndKeys.ords, 0, null, 1, null, null); + assertKeys(ordsAndKeys.keys, new Object[][] { new Object[] { 1L, "cat" }, new Object[] { 0L, "dog" } }); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 2))); + } + }, b1, b2); } - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } private void append(LongBlock.Builder b1, BytesRefBlock.Builder b2, long[] v1, String[] v2) { @@ -1003,123 +959,128 @@ private void append(LongBlock.Builder b1, BytesRefBlock.Builder b2, long[] v1, S } public void testLongBytesRefHashWithMultiValuedFields() { - var b1 = BlockFactory.getNonBreakingInstance().newLongBlockBuilder(8); - var b2 = BlockFactory.getNonBreakingInstance().newBytesRefBlockBuilder(8); - append(b1, b2, new long[] { 1, 2 }, new String[] { "a", "b" }); - append(b1, b2, new long[] { 1, 2 }, new String[] { "a" }); - append(b1, b2, new long[] { 1 }, new String[] { "a", "b" }); - append(b1, b2, new long[] { 1 }, new String[] { "a" }); - append(b1, b2, null, new String[] { "a" }); - append(b1, b2, new long[] { 1 }, null); - append(b1, b2, new long[] { 1, 1, 1 }, new String[] { "a", "a", "a" }); - append(b1, b2, new long[] { 1, 1, 2, 2 }, new String[] { "a", "b", "b" }); - append(b1, b2, new long[] { 1, 2, 3 }, new String[] { "c", "c", "a" }); - - OrdsAndKeys ordsAndKeys = hash(b1.build(), b2.build()); - if (forcePackedHash) { - assertThat(ordsAndKeys.description, startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:BYTES_REF], entries=10, size=")); - assertOrds( - ordsAndKeys.ords, - new int[] { 0, 1, 2, 3 }, - new int[] { 0, 2 }, - new int[] { 0, 1 }, - new int[] { 0 }, - new int[] { 4 }, - new int[] { 5 }, - new int[] { 0 }, - new int[] { 0, 1, 2, 3 }, - new int[] { 6, 0, 7, 2, 8, 9 } - ); - assertKeys( - ordsAndKeys.keys, - new Object[][] { - new Object[] { 1L, "a" }, - new Object[] { 1L, "b" }, - new Object[] { 2L, "a" }, - new Object[] { 2L, "b" }, - new Object[] { null, "a" }, - new Object[] { 1L, null }, - new Object[] { 1L, "c" }, - new Object[] { 2L, "c" }, - new Object[] { 3L, "c" }, - new Object[] { 3L, "a" }, } - ); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 10))); - } else { - assertThat( - ordsAndKeys.description, - equalTo("BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=8, size=491b}") - ); - assertOrds( - ordsAndKeys.ords, - new int[] { 0, 1, 2, 3 }, - new int[] { 0, 1 }, - new int[] { 0, 2 }, - new int[] { 0 }, - null, - null, - new int[] { 0 }, - new int[] { 0, 1, 2, 3 }, - new int[] { 4, 5, 6, 0, 1, 7 } - ); - assertKeys( - ordsAndKeys.keys, - new Object[][] { - new Object[] { 1L, "a" }, - new Object[] { 2L, "a" }, - new Object[] { 1L, "b" }, - new Object[] { 2L, "b" }, - new Object[] { 1L, "c" }, - new Object[] { 2L, "c" }, - new Object[] { 3L, "c" }, - new Object[] { 3L, "a" }, } - ); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 8))); + try ( + LongBlock.Builder b1 = blockFactory.newLongBlockBuilder(8); + BytesRefBlock.Builder b2 = blockFactory.newBytesRefBlockBuilder(8) + ) { + append(b1, b2, new long[] { 1, 2 }, new String[] { "a", "b" }); + append(b1, b2, new long[] { 1, 2 }, new String[] { "a" }); + append(b1, b2, new long[] { 1 }, new String[] { "a", "b" }); + append(b1, b2, new long[] { 1 }, new String[] { "a" }); + append(b1, b2, null, new String[] { "a" }); + append(b1, b2, new long[] { 1 }, null); + append(b1, b2, new long[] { 1, 1, 1 }, new String[] { "a", "a", "a" }); + append(b1, b2, new long[] { 1, 1, 2, 2 }, new String[] { "a", "b", "b" }); + append(b1, b2, new long[] { 1, 2, 3 }, new String[] { "c", "c", "a" }); + + hash((OrdsAndKeys ordsAndKeys) -> { + if (forcePackedHash) { + assertThat( + ordsAndKeys.description, + startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:BYTES_REF], entries=10, size=") + ); + assertOrds( + ordsAndKeys.ords, + new int[] { 0, 1, 2, 3 }, + new int[] { 0, 2 }, + new int[] { 0, 1 }, + new int[] { 0 }, + new int[] { 4 }, + new int[] { 5 }, + new int[] { 0 }, + new int[] { 0, 1, 2, 3 }, + new int[] { 6, 0, 7, 2, 8, 9 } + ); + assertKeys( + ordsAndKeys.keys, + new Object[][] { + new Object[] { 1L, "a" }, + new Object[] { 1L, "b" }, + new Object[] { 2L, "a" }, + new Object[] { 2L, "b" }, + new Object[] { null, "a" }, + new Object[] { 1L, null }, + new Object[] { 1L, "c" }, + new Object[] { 2L, "c" }, + new Object[] { 3L, "c" }, + new Object[] { 3L, "a" }, } + ); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 10))); + } else { + assertThat( + ordsAndKeys.description, + equalTo("BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=8, size=491b}") + ); + assertOrds( + ordsAndKeys.ords, + new int[] { 0, 1, 2, 3 }, + new int[] { 0, 1 }, + new int[] { 0, 2 }, + new int[] { 0 }, + null, + null, + new int[] { 0 }, + new int[] { 0, 1, 2, 3 }, + new int[] { 4, 5, 6, 0, 1, 7 } + ); + assertKeys( + ordsAndKeys.keys, + new Object[][] { + new Object[] { 1L, "a" }, + new Object[] { 2L, "a" }, + new Object[] { 1L, "b" }, + new Object[] { 2L, "b" }, + new Object[] { 1L, "c" }, + new Object[] { 2L, "c" }, + new Object[] { 3L, "c" }, + new Object[] { 3L, "a" }, } + ); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, 8))); + } + }, b1, b2); } - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - assertThat(breaker.getUsed(), is(0L)); } public void testBytesRefLongHashHugeCombinatorialExplosion() { long[] v1 = LongStream.range(0, 3000).toArray(); String[] v2 = LongStream.range(100, 200).mapToObj(l -> "a" + l).toArray(String[]::new); - var b1 = BlockFactory.getNonBreakingInstance().newLongBlockBuilder(v1.length); - var b2 = BlockFactory.getNonBreakingInstance().newBytesRefBlockBuilder(v2.length); - append(b1, b2, v1, v2); - - int[] expectedEntries = new int[1]; - int pageSize = between(1000, 16 * 1024); - hash(ordsAndKeys -> { - int start = expectedEntries[0]; - expectedEntries[0] = Math.min(expectedEntries[0] + pageSize, v1.length * v2.length); - assertThat( - ordsAndKeys.description, - forcePackedHash - ? startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:BYTES_REF], entries=" + expectedEntries[0] + ", size=") - : startsWith( - "BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=" + expectedEntries[0] + ", size=" - ) - ); - assertOrds(ordsAndKeys.ords, IntStream.range(start, expectedEntries[0]).toArray()); - assertKeys( - ordsAndKeys.keys, - IntStream.range(0, expectedEntries[0]) - .mapToObj( - i -> forcePackedHash - ? new Object[] { v1[i / v2.length], v2[i % v2.length] } - : new Object[] { v1[i % v1.length], v2[i / v1.length] } - ) - .toArray(l -> new Object[l][]) - ); - assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, expectedEntries[0]))); - Releasables.closeExpectNoException(ordsAndKeys.keys); - Releasables.closeExpectNoException(ordsAndKeys.nonEmpty); - }, pageSize, b1.build(), b2.build()); + try ( + LongBlock.Builder b1 = blockFactory.newLongBlockBuilder(v1.length); + BytesRefBlock.Builder b2 = blockFactory.newBytesRefBlockBuilder(v2.length); + ) { + append(b1, b2, v1, v2); + int[] expectedEntries = new int[1]; + int pageSize = between(1000, 16 * 1024); + hash(ordsAndKeys -> { + int start = expectedEntries[0]; + expectedEntries[0] = Math.min(expectedEntries[0] + pageSize, v1.length * v2.length); + assertThat( + ordsAndKeys.description, + forcePackedHash + ? startsWith("PackedValuesBlockHash{groups=[0:LONG, 1:BYTES_REF], entries=" + expectedEntries[0] + ", size=") + : startsWith( + "BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=" + + expectedEntries[0] + + ", size=" + ) + ); + assertOrds(ordsAndKeys.ords, IntStream.range(start, expectedEntries[0]).toArray()); + assertKeys( + ordsAndKeys.keys, + IntStream.range(0, expectedEntries[0]) + .mapToObj( + i -> forcePackedHash + ? new Object[] { v1[i / v2.length], v2[i % v2.length] } + : new Object[] { v1[i % v1.length], v2[i / v1.length] } + ) + .toArray(l -> new Object[l][]) + ); + assertThat(ordsAndKeys.nonEmpty, equalTo(intRange(0, expectedEntries[0]))); + }, pageSize, b1, b2); - assertThat("misconfigured test", expectedEntries[0], greaterThan(0)); - assertThat(breaker.getUsed(), is(0L)); + assertThat("misconfigured test", expectedEntries[0], greaterThan(0)); + } } record OrdsAndKeys(String description, int positionOffset, IntBlock ords, Block[] keys, IntVector nonEmpty) {} @@ -1128,29 +1089,53 @@ record OrdsAndKeys(String description, int positionOffset, IntBlock ords, Block[ * Hash some values into a single block of group ids. If the hash produces * more than one block of group ids this will fail. */ - private OrdsAndKeys hash(Block... values) { - OrdsAndKeys[] result = new OrdsAndKeys[1]; + private void hash(Consumer callback, Block.Builder... values) { + Block[] blocks = new Block[values.length]; + for (int i = 0; i < blocks.length; i++) { + blocks[i] = values[i].build(); + } + hash(callback, blocks); + } + + /** + * Hash some values into a single block of group ids. If the hash produces + * more than one block of group ids this will fail. + */ + private void hash(Consumer callback, Block... values) { + boolean[] called = new boolean[] { false }; hash(ordsAndKeys -> { - if (result[0] != null) { + if (called[0]) { throw new IllegalStateException("hash produced more than one block"); } - result[0] = ordsAndKeys; + called[0] = true; + callback.accept(ordsAndKeys); }, 16 * 1024, values); - return result[0]; } - private void hash(Consumer callback, int emitBatchSize, Block... values) { - List specs = new ArrayList<>(values.length); - for (int c = 0; c < values.length; c++) { - specs.add(new HashAggregationOperator.GroupSpec(c, values[c].elementType())); + private void hash(Consumer callback, int emitBatchSize, Block.Builder... values) { + Block[] blocks = new Block[values.length]; + for (int i = 0; i < blocks.length; i++) { + blocks[i] = values[i].build(); } - DriverContext driverContext = new DriverContext(bigArrays, blockFactory); - try ( - BlockHash blockHash = forcePackedHash - ? new PackedValuesBlockHash(specs, driverContext, emitBatchSize) - : BlockHash.build(specs, driverContext, emitBatchSize, true) - ) { - hash(true, blockHash, callback, values); + hash(callback, emitBatchSize, blocks); + } + + private void hash(Consumer callback, int emitBatchSize, Block... values) { + try { + List specs = new ArrayList<>(values.length); + for (int c = 0; c < values.length; c++) { + specs.add(new HashAggregationOperator.GroupSpec(c, values[c].elementType())); + } + DriverContext driverContext = new DriverContext(bigArrays, blockFactory); + try ( + BlockHash blockHash = forcePackedHash + ? new PackedValuesBlockHash(specs, driverContext, emitBatchSize) + : BlockHash.build(specs, driverContext, emitBatchSize, true) + ) { + hash(true, blockHash, callback, values); + } + } finally { + Releasables.closeExpectNoException(values); } } @@ -1166,24 +1151,28 @@ public void add(int positionOffset, IntBlock groupIds) { blockHash.nonEmpty() ); - Set allowedOrds = new HashSet<>(); - for (int p = 0; p < result.nonEmpty.getPositionCount(); p++) { - allowedOrds.add(result.nonEmpty.getInt(p)); - } - for (int p = 0; p < result.ords.getPositionCount(); p++) { - if (result.ords.isNull(p)) { - continue; + try { + Set allowedOrds = new HashSet<>(); + for (int p = 0; p < result.nonEmpty.getPositionCount(); p++) { + allowedOrds.add(result.nonEmpty.getInt(p)); } - int start = result.ords.getFirstValueIndex(p); - int end = start + result.ords.getValueCount(p); - for (int i = start; i < end; i++) { - int ord = result.ords.getInt(i); - if (false == allowedOrds.contains(ord)) { - fail("ord is not allowed " + ord); + for (int p = 0; p < result.ords.getPositionCount(); p++) { + if (result.ords.isNull(p)) { + continue; + } + int start = result.ords.getFirstValueIndex(p); + int end = start + result.ords.getValueCount(p); + for (int i = start; i < end; i++) { + int ord = result.ords.getInt(i); + if (false == allowedOrds.contains(ord)) { + fail("ord is not allowed " + ord); + } } } + callback.accept(result); + } finally { + Releasables.close(result.keys == null ? null : Releasables.wrap(result.keys), result.nonEmpty); } - callback.accept(result); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/MockBlockFactory.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/MockBlockFactory.java index 6ba2482ab6d8d..0a63043ecc14b 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/MockBlockFactory.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/MockBlockFactory.java @@ -146,7 +146,8 @@ public BooleanBlock.Builder newBooleanBlockBuilder(int estimatedSize) { return b; } - BooleanVector.FixedBuilder newBooleanVectorFixedBuilder(int size) { + @Override + public BooleanVector.FixedBuilder newBooleanVectorFixedBuilder(int size) { var b = super.newBooleanVectorFixedBuilder(size); track(b, trackDetail()); return b;