ESQL: Fixed length vector builder

x-pack/plugin/esql/compute/build.gradle

@@ -343,6 +343,27 @@ tasks.named('stringTemplates').configure {
     it.inputFile =  vectorBuildersInputFile
     it.outputFile = "org/elasticsearch/compute/data/BooleanVectorBuilder.java"
   }
+  File vectorFixedBuildersInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-VectorFixedBuilder.java.st")
+  template {
+    it.properties = intProperties
+    it.inputFile =  vectorFixedBuildersInputFile
+    it.outputFile = "org/elasticsearch/compute/data/IntVectorFixedBuilder.java"
+  }
+  template {
+    it.properties = longProperties
+    it.inputFile =  vectorFixedBuildersInputFile
+    it.outputFile = "org/elasticsearch/compute/data/LongVectorFixedBuilder.java"
+  }
+  template {
+    it.properties = doubleProperties
+    it.inputFile =  vectorFixedBuildersInputFile
+    it.outputFile = "org/elasticsearch/compute/data/DoubleVectorFixedBuilder.java"
+  }
+  template {
+    it.properties = booleanProperties
+    it.inputFile =  vectorFixedBuildersInputFile
+    it.outputFile = "org/elasticsearch/compute/data/BooleanVectorFixedBuilder.java"
+  }
   File stateInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-State.java.st")
   template {
     it.properties = intProperties

x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/MvEvaluatorImplementer.java

@@ -31,17 +31,15 @@
 import static org.elasticsearch.compute.gen.Methods.getMethod;
 import static org.elasticsearch.compute.gen.Types.ABSTRACT_MULTIVALUE_FUNCTION_EVALUATOR;
 import static org.elasticsearch.compute.gen.Types.ABSTRACT_NULLABLE_MULTIVALUE_FUNCTION_EVALUATOR;
-import static org.elasticsearch.compute.gen.Types.BIG_ARRAYS;
 import static org.elasticsearch.compute.gen.Types.BLOCK;
 import static org.elasticsearch.compute.gen.Types.BYTES_REF;
-import static org.elasticsearch.compute.gen.Types.BYTES_REF_ARRAY;
 import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT;
 import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR;
 import static org.elasticsearch.compute.gen.Types.SOURCE;
 import static org.elasticsearch.compute.gen.Types.VECTOR;
 import static org.elasticsearch.compute.gen.Types.WARNINGS;
-import static org.elasticsearch.compute.gen.Types.arrayVectorType;
 import static org.elasticsearch.compute.gen.Types.blockType;
+import static org.elasticsearch.compute.gen.Types.vectorType;
 
 public class MvEvaluatorImplementer {
     private final TypeElement declarationType;
@@ -197,16 +195,25 @@ private MethodSpec evalShell(
         builder.addStatement("int positionCount = v.getPositionCount()");
         if (nullable) {
             TypeName resultBlockType = blockType(resultType);
-            builder.addStatement("$T.Builder builder = $T.newBlockBuilder(positionCount)", resultBlockType, resultBlockType);
+            builder.addStatement(
+                "$T.Builder builder = $T.newBlockBuilder(positionCount, driverContext.blockFactory())",
+                resultBlockType,
+                resultBlockType
+            );
         } else if (resultType.equals(BYTES_REF)) {
+            TypeName resultVectorType = vectorType(resultType);
             builder.addStatement(
-                "$T values = new $T(positionCount, $T.NON_RECYCLING_INSTANCE)",  // TODO blocks should use recycling array
-                BYTES_REF_ARRAY,
-                BYTES_REF_ARRAY,
-                BIG_ARRAYS
+                "$T.Builder builder = $T.newVectorBuilder(positionCount, driverContext.blockFactory())",
+                resultVectorType,
+                resultVectorType
             );
         } else {
-            builder.addStatement("$T[] values = new $T[positionCount]", resultType, resultType);
+            TypeName resultVectorType = vectorType(resultType);
+            builder.addStatement(
+                "$T.FixedBuilder builder = $T.newVectorFixedBuilder(positionCount, driverContext.blockFactory())",
+                resultVectorType,
+                resultVectorType
+            );
         }
 
         if (false == workType.equals(fieldType) && workType.isPrimitive() == false) {
@@ -244,11 +251,7 @@ private MethodSpec evalShell(
         }
         builder.endControlFlow();
 
-        if (nullable) {
-            builder.addStatement("return builder.build()");
-        } else {
-            builder.addStatement("return new $T(values, positionCount)", arrayVectorType(resultType));
-        }
+        builder.addStatement("return builder.build()");
         return builder.build();
     }
 
@@ -268,7 +271,7 @@ private MethodSpec eval(String name, boolean nullable) {
                 builder.beginControlFlow("if (valueCount == 1)");
                 fetch(builder, "value", fieldType, "first", workType.equals(fieldType) ? "firstScratch" : "valueScratch");
                 singleValueFunction.call(builder);
-                writeResult(builder, nullable);
+                writeResult(builder);
                 builder.addStatement("continue");
                 builder.endControlFlow();
             }
@@ -302,7 +305,7 @@ private MethodSpec eval(String name, boolean nullable) {
                 builder.endControlFlow();
                 finishFunction.call(builder, "work");
             }
-            writeResult(builder, nullable);
+            writeResult(builder);
         });
     }
 
@@ -313,7 +316,7 @@ private MethodSpec evalSingleValued(String name, boolean nullable) {
             builder.addStatement("int first = v.getFirstValueIndex(p)");
             fetch(builder, "value", fieldType, "first", workType.equals(fieldType) ? "firstScratch" : "valueScratch");
             singleValueFunction.call(builder);
-            writeResult(builder, nullable);
+            writeResult(builder);
         });
     }
 
@@ -332,17 +335,15 @@ private MethodSpec evalAscending(String name, boolean nullable) {
         return evalShell(name, false, nullable, javadoc, builder -> {}, builder -> {
             builder.addStatement("int first = v.getFirstValueIndex(p)");
             ascendingFunction.call(builder);
-            writeResult(builder, nullable);
+            writeResult(builder);
         });
     }
 
-    private void writeResult(MethodSpec.Builder builder, boolean nullable) {
-        if (nullable) {
-            builder.addStatement("builder.$L(result)", appendMethod(resultType));
-        } else if (fieldType.equals(BYTES_REF)) {
-            builder.addStatement("values.append(result)");
+    private void writeResult(MethodSpec.Builder builder) {
+        if (fieldType.equals(BYTES_REF)) {
+            builder.addStatement("builder.appendBytesRef(result)");
         } else {
-            builder.addStatement("values[p] = result");
+            builder.addStatement("builder.$L(result)", appendMethod(resultType));
         }
     }
 

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java

@@ -17,7 +17,7 @@
  */
 public final class BooleanArrayVector extends AbstractVector implements BooleanVector {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BooleanArrayVector.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BooleanArrayVector.class);
 
     private final boolean[] values;
 

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java

@@ -106,10 +106,25 @@ static Builder newVectorBuilder(int estimatedSize) {
         return newVectorBuilder(estimatedSize, BlockFactory.getNonBreakingInstance());
     }
 
+    /**
+     * Creates a builder that grows as needed. Prefer {@link #newVectorFixedBuilder}
+     * if you know the size up front because it's faster.
+     */
     static Builder newVectorBuilder(int estimatedSize, BlockFactory blockFactory) {
         return blockFactory.newBooleanVectorBuilder(estimatedSize);
     }
 
+    /**
+     * Creates a builder that never grows. Prefer this over {@link #newVectorBuilder}
+     * if you know the size up front because it's faster.
+     */
+    static FixedBuilder newVectorFixedBuilder(int size, BlockFactory blockFactory) {
+        return blockFactory.newBooleanVectorFixedBuilder(size);
+    }
+
+    /**
+     * A builder that grows as needed.
+     */
     sealed interface Builder extends Vector.Builder permits BooleanVectorBuilder {
         /**
          * Appends a boolean to the current entry.
@@ -119,4 +134,17 @@ sealed interface Builder extends Vector.Builder permits BooleanVectorBuilder {
         @Override
         BooleanVector build();
     }
+
+    /**
+     * A builder that never grows.
+     */
+    sealed interface FixedBuilder extends Vector.Builder permits BooleanVectorFixedBuilder {
+        /**
+         * Appends a boolean to the current entry.
+         */
+        FixedBuilder appendBoolean(boolean value);
+
+        @Override
+        BooleanVector build();
+    }
 }

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBuilder.java

@@ -10,7 +10,7 @@
 import java.util.Arrays;
 
 /**
- * Block build of BooleanBlocks.
+ * Builder for {@link BooleanVector}s that grows as needed.
  * This class is generated. Do not edit it.
  */
 final class BooleanVectorBuilder extends AbstractVectorBuilder implements BooleanVector.Builder {

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorFixedBuilder.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.data;
+
+import org.apache.lucene.util.RamUsageEstimator;
+
+/**
+ * Builder for {@link BooleanVector}s that never grows. Prefer this to
+ * {@link BooleanVectorBuilder} if you know the precise size up front because
+ * it's faster.
+ * This class is generated. Do not edit it.
+ */
+final class BooleanVectorFixedBuilder implements BooleanVector.FixedBuilder {
+    private final BlockFactory blockFactory;
+    private final boolean[] values;
+    /**
+     * The next value to write into. {@code -1} means the vector has already
+     * been built.
+     */
+    private int nextIndex;
+
+    BooleanVectorFixedBuilder(int size, BlockFactory blockFactory) {
+        blockFactory.adjustBreaker(ramBytesUsed(size), false);
+        this.blockFactory = blockFactory;
+        this.values = new boolean[size];
+    }
+
+    @Override
+    public BooleanVectorFixedBuilder appendBoolean(boolean value) {
+        values[nextIndex++] = value;
+        return this;
+    }
+
+    private static long ramBytesUsed(int size) {
+        return size == 1
+            ? ConstantBooleanVector.RAM_BYTES_USED
+            : BooleanArrayVector.BASE_RAM_BYTES_USED + RamUsageEstimator.alignObjectSize(
+                (long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + size * Byte.BYTES
+            );
+    }
+
+    @Override
+    public BooleanVector build() {
+        if (nextIndex < 0) {
+            throw new IllegalStateException("already closed");
+        }
+        if (nextIndex != values.length) {
+            throw new IllegalStateException("expected to write [" + values.length + "] entries but wrote [" + nextIndex + "]");
+        }
+        nextIndex = -1;
+        if (values.length == 1) {
+            return new ConstantBooleanVector(values[0], 1, blockFactory);
+        }
+        return new BooleanArrayVector(values, values.length, blockFactory);
+    }
+}

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java

@@ -18,7 +18,7 @@
  */
 public final class BytesRefArrayVector extends AbstractVector implements BytesRefVector {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BytesRefArrayVector.class);
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BytesRefArrayVector.class);
 
     private final BytesRefArray values;
 

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVector.java

@@ -106,10 +106,16 @@ static Builder newVectorBuilder(int estimatedSize) {
         return newVectorBuilder(estimatedSize, BlockFactory.getNonBreakingInstance());
     }
 
+    /**
+     * Creates a builder that grows as needed.
+     */
     static Builder newVectorBuilder(int estimatedSize, BlockFactory blockFactory) {
         return blockFactory.newBytesRefVectorBuilder(estimatedSize);
     }
 
+    /**
+     * A builder that grows as needed.
+     */
     sealed interface Builder extends Vector.Builder permits BytesRefVectorBuilder {
         /**
          * Appends a BytesRef to the current entry.
@@ -119,4 +125,5 @@ sealed interface Builder extends Vector.Builder permits BytesRefVectorBuilder {
         @Override
         BytesRefVector build();
     }
+
 }

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBuilder.java

@@ -13,7 +13,7 @@
 import org.elasticsearch.core.Releasables;
 
 /**
- * Block build of BytesRefBlocks.
+ * Builder for {@link BytesRefVector}s that grows as needed.
  * This class is generated. Do not edit it.
  */
 final class BytesRefVectorBuilder extends AbstractVectorBuilder implements BytesRefVector.Builder {

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java

@@ -15,7 +15,7 @@
  */
 public final class ConstantBooleanVector extends AbstractVector implements BooleanVector {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantBooleanVector.class);
+    static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantBooleanVector.class);
 
     private final boolean value;
 
@@ -55,7 +55,7 @@ public boolean isConstant() {
 
     @Override
     public long ramBytesUsed() {
-        return BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOfInstance(boolean.class);
+        return RAM_BYTES_USED;
     }
 
     @Override

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBytesRefVector.java

@@ -16,8 +16,8 @@
  */
 public final class ConstantBytesRefVector extends AbstractVector implements BytesRefVector {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantBytesRefVector.class);
-
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantBytesRefVector.class) + RamUsageEstimator
+        .shallowSizeOfInstance(BytesRef.class);
     private final BytesRef value;
 
     public ConstantBytesRefVector(BytesRef value, int positionCount) {
@@ -56,7 +56,7 @@ public boolean isConstant() {
 
     @Override
     public long ramBytesUsed() {
-        return BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class);
+        return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(value.bytes);
     }
 
     @Override

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantDoubleVector.java

@@ -15,7 +15,7 @@
  */
 public final class ConstantDoubleVector extends AbstractVector implements DoubleVector {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantDoubleVector.class);
+    static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantDoubleVector.class);
 
     private final double value;
 
@@ -55,7 +55,7 @@ public boolean isConstant() {
 
     @Override
     public long ramBytesUsed() {
-        return BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOfInstance(double.class);
+        return RAM_BYTES_USED;
     }
 
     @Override

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantIntVector.java

@@ -15,7 +15,7 @@
  */
 public final class ConstantIntVector extends AbstractVector implements IntVector {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantIntVector.class);
+    static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantIntVector.class);
 
     private final int value;
 
@@ -55,7 +55,7 @@ public boolean isConstant() {
 
     @Override
     public long ramBytesUsed() {
-        return BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOfInstance(int.class);
+        return RAM_BYTES_USED;
     }
 
     @Override

x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantLongVector.java

@@ -15,7 +15,7 @@
  */
 public final class ConstantLongVector extends AbstractVector implements LongVector {
 
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantLongVector.class);
+    static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantLongVector.class);
 
     private final long value;
 
@@ -55,7 +55,7 @@ public boolean isConstant() {
 
     @Override
     public long ramBytesUsed() {
-        return BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOfInstance(long.class);
+        return RAM_BYTES_USED;
     }
 
     @Override