diff --git a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java
index 208d29edad71d..288462ba3bbcb 100644
--- a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java
+++ b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java
@@ -48,7 +48,7 @@ public BytesRefHash(long capacity, float maxLoadFactor, BigArrays bigArrays) {
         boolean success = false;
         try {
             // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
-            this.hashes = bigArrays.newIntArray(capacity, false);
+            this.hashes = bigArrays.newIntArray(maxSize, false);
             this.bytesRefs = new BytesRefArray(capacity, bigArrays);
             success = true;
         } finally {
@@ -98,7 +98,7 @@ public BytesRefHash(BytesRefArray bytesRefs, float maxLoadFactor, BigArrays bigA
         boolean success = false;
         try {
             // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
-            this.hashes = bigArrays.newIntArray(bytesRefs.size() + 1, false);
+            this.hashes = bigArrays.newIntArray(maxSize, false);
             this.bytesRefs = BytesRefArray.takeOwnershipOf(bytesRefs);
             success = true;
         } finally {
@@ -182,7 +182,6 @@ private long set(BytesRef key, int code, long id) {
     private void append(long id, BytesRef key, int code) {
         assert size == id;
         bytesRefs.append(key);
-        hashes = bigArrays.grow(hashes, id + 1);
         hashes.set(id, code);
     }
 
@@ -211,6 +210,7 @@ public long add(BytesRef key, int code) {
         if (size >= maxSize) {
             assert size == maxSize;
             grow();
+            hashes = bigArrays.resize(hashes, maxSize);
         }
         assert size < maxSize;
         return set(key, rehash(code), size);
diff --git a/server/src/main/java/org/elasticsearch/common/util/LongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongHash.java
index 0c681063c50b0..3eeb60e419a19 100644
--- a/server/src/main/java/org/elasticsearch/common/util/LongHash.java
+++ b/server/src/main/java/org/elasticsearch/common/util/LongHash.java
@@ -33,7 +33,7 @@ public LongHash(long capacity, float maxLoadFactor, BigArrays bigArrays) {
         super(capacity, maxLoadFactor, bigArrays);
         try {
             // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
-            keys = bigArrays.newLongArray(capacity, false);
+            keys = bigArrays.newLongArray(maxSize, false);
         } finally {
             if (keys == null) {
                 close();
@@ -78,7 +78,6 @@ private long set(long key, long id) {
     }
 
     private void append(long id, long key) {
-        keys = bigArrays.grow(keys, id + 1);
         keys.set(id, key);
     }
 
@@ -102,6 +101,7 @@ public long add(long key) {
         if (size >= maxSize) {
             assert size == maxSize;
             grow();
+            keys = bigArrays.resize(keys, maxSize);
         }
         assert size < maxSize;
         return set(key, size);
diff --git a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java
index f7708af59dde2..031794ed9c9c6 100644
--- a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java
+++ b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java
@@ -40,7 +40,7 @@ public LongLongHash(long capacity, float maxLoadFactor, BigArrays bigArrays) {
         super(capacity, maxLoadFactor, bigArrays);
         try {
             // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
-            keys = bigArrays.newLongArray(2 * capacity, false);
+            keys = bigArrays.newLongArray(2 * maxSize, false);
         } finally {
             if (keys == null) {
                 close();
@@ -99,7 +99,6 @@ private long set(long key1, long key2, long id) {
 
     private void append(long id, long key1, long key2) {
         long keyOffset = 2 * id;
-        keys = bigArrays.grow(keys, keyOffset + 2);
         keys.set(keyOffset, key1);
         keys.set(keyOffset + 1, key2);
     }
@@ -128,6 +127,7 @@ public long add(long key1, long key2) {
         if (size >= maxSize) {
             assert size == maxSize;
             grow();
+            keys = bigArrays.resize(keys, maxSize * 2);
         }
         assert size < maxSize;
         return set(key1, key2, size);
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java
index aeea18e52da0f..088e791348840 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java
@@ -1147,7 +1147,7 @@ public void testLongBytesRefHashWithMultiValuedFields() {
                 } else {
                     assertThat(
                         ordsAndKeys.description,
-                        equalTo("BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=9, size=491b}")
+                        equalTo("BytesRefLongBlockHash{keys=[BytesRefKey[channel=1], LongKey[channel=0]], entries=9, size=483b}")
                     );
                     assertOrds(
                         ordsAndKeys.ords,