From 2c88a89c50da01c060c1b5e4faec4ee7a2cf098f Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Thu, 14 Sep 2023 17:35:06 +0100
Subject: [PATCH 01/41] Drop unnecessary exception mangling (#99560)

Today the ESQL module uses `ListenableActionFuture` throughout, but this
is just the same as `SubscribableListener` except for the way it mangles
exceptions. The exception-mangling behaviour is unnecessary, so this
commit removes it.
---
 .../action/support/SubscribableListener.java  | 40 ++++++++++++++++++-
 .../support/SubscribableListenerTests.java    | 17 ++++++++
 .../compute/operator/AsyncOperator.java       | 10 ++---
 .../compute/operator/Driver.java              | 20 +++++-----
 .../compute/operator/Operator.java            | 12 ++----
 .../operator/exchange/ExchangeBuffer.java     | 20 +++++-----
 .../operator/exchange/ExchangeSink.java       |  4 +-
 .../exchange/ExchangeSinkHandler.java         |  9 ++---
 .../exchange/ExchangeSinkOperator.java        |  4 +-
 .../operator/exchange/ExchangeSource.java     |  4 +-
 .../exchange/ExchangeSourceHandler.java       |  8 ++--
 .../exchange/ExchangeSourceOperator.java      |  6 +--
 .../compute/operator/AsyncOperatorTests.java  |  4 +-
 .../exchange/ExchangeServiceTests.java        |  4 +-
 .../xpack/esql/plugin/ComputeService.java     |  4 +-
 15 files changed, 106 insertions(+), 60 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java b/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java
index 96b54a951ccc9..5ba43111b4f03 100644
--- a/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java
+++ b/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
+import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -29,7 +30,7 @@
 /**
  * An {@link ActionListener} to which other {@link ActionListener} instances can subscribe, such that when this listener is completed it
  * fans-out its result to the subscribed listeners.
- *
+ * <p>
  * Similar to {@link ListenableActionFuture} and {@link ListenableFuture} except for its handling of exceptions: if this listener is
  * completed exceptionally then the exception is passed to subscribed listeners without modification.
  */
@@ -38,6 +39,41 @@ public class SubscribableListener<T> implements ActionListener<T> {
     private static final Logger logger = LogManager.getLogger(SubscribableListener.class);
     private static final Object EMPTY = new Object();
 
+    /**
+     * Create a {@link SubscribableListener} which is incomplete.
+     */
+    public SubscribableListener() {
+        this(EMPTY);
+    }
+
+    /**
+     * Create a {@link SubscribableListener} which has already succeeded with the given result.
+     */
+    public static <T> SubscribableListener<T> newSucceeded(T result) {
+        return new SubscribableListener<>(new SuccessResult<>(result));
+    }
+
+    /**
+     * Create a {@link SubscribableListener} which has already failed with the given exception.
+     */
+    public static <T> SubscribableListener<T> newFailed(Exception exception) {
+        return new SubscribableListener<>(new FailureResult(exception, exception));
+    }
+
+    /**
+     * Create a {@link SubscribableListener}, fork a computation to complete it, and return the listener. If the forking itself throws an
+     * exception then the exception is caught and fed to the returned listener.
+     */
+    public static <T> SubscribableListener<T> newForked(CheckedConsumer<ActionListener<T>, ? extends Exception> fork) {
+        final var listener = new SubscribableListener<T>();
+        ActionListener.run(listener, fork::accept);
+        return listener;
+    }
+
+    private SubscribableListener(Object initialState) {
+        state = initialState;
+    }
+
     /**
      * If we are incomplete, {@code state} may be one of the following depending on how many waiting subscribers there are:
      * <ul>
@@ -50,7 +86,7 @@ public class SubscribableListener<T> implements ActionListener<T> {
      * subsequent subscribers.
      */
     @SuppressWarnings("FieldMayBeFinal") // updated via VH_STATE_FIELD (and _only_ via VH_STATE_FIELD)
-    private volatile Object state = EMPTY;
+    private volatile Object state;
 
     /**
      * Add a listener to this listener's collection of subscribers. If this listener is complete, this method completes the subscribing
diff --git a/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java
index 91404b3c72cb7..9ef5e40e1f932 100644
--- a/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java
@@ -381,4 +381,21 @@ public void onFailure(Exception e) {
         assertTrue(listener.isDone());
     }
 
+    public void testCreateUtils() throws Exception {
+        final var succeeded = SubscribableListener.newSucceeded("result");
+        assertTrue(succeeded.isDone());
+        assertEquals("result", succeeded.rawResult());
+
+        final var failed = SubscribableListener.newFailed(new ElasticsearchException("simulated"));
+        assertTrue(failed.isDone());
+        assertEquals("simulated", expectThrows(ElasticsearchException.class, failed::rawResult).getMessage());
+
+        final var forkedListenerRef = new AtomicReference<ActionListener<Object>>();
+        final var forked = SubscribableListener.newForked(forkedListenerRef::set);
+        assertSame(forked, forkedListenerRef.get());
+
+        final var forkFailed = SubscribableListener.newForked(l -> { throw new ElasticsearchException("simulated fork failure"); });
+        assertTrue(forkFailed.isDone());
+        assertEquals("simulated fork failure", expectThrows(ElasticsearchException.class, forkFailed::rawResult).getMessage());
+    }
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java
index 1258e2b4c7177..34fdd32aab98a 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ListenableActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.index.seqno.LocalCheckpointTracker;
@@ -26,7 +26,7 @@
  */
 public abstract class AsyncOperator implements Operator {
 
-    private volatile ListenableActionFuture<Void> blockedFuture;
+    private volatile SubscribableListener<Void> blockedFuture;
 
     private final Map<Long, Page> buffers = ConcurrentCollections.newConcurrentMap();
     private final AtomicReference<Exception> failure = new AtomicReference<>();
@@ -109,7 +109,7 @@ private void onSeqNoCompleted(long seqNo) {
 
     private void notifyIfBlocked() {
         if (blockedFuture != null) {
-            final ListenableActionFuture<Void> future;
+            final SubscribableListener<Void> future;
             synchronized (this) {
                 future = blockedFuture;
                 this.blockedFuture = null;
@@ -153,7 +153,7 @@ public Page getOutput() {
     }
 
     @Override
-    public ListenableActionFuture<Void> isBlocked() {
+    public SubscribableListener<Void> isBlocked() {
         if (finished) {
             return Operator.NOT_BLOCKED;
         }
@@ -167,7 +167,7 @@ public ListenableActionFuture<Void> isBlocked() {
                 return Operator.NOT_BLOCKED;
             }
             if (blockedFuture == null) {
-                blockedFuture = new ListenableActionFuture<>();
+                blockedFuture = new SubscribableListener<>();
             }
             return blockedFuture;
         }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java
index 5f5cbe2707fe3..281693a487255 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Driver.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.compute.operator;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ListenableActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.compute.Describable;
 import org.elasticsearch.compute.data.Page;
@@ -53,7 +53,7 @@ public class Driver implements Releasable, Describable {
     private final long statusNanos;
 
     private final AtomicReference<String> cancelReason = new AtomicReference<>();
-    private final AtomicReference<ListenableActionFuture<Void>> blocked = new AtomicReference<>();
+    private final AtomicReference<SubscribableListener<Void>> blocked = new AtomicReference<>();
     /**
      * Status reported to the tasks API. We write the status at most once every
      * {@link #statusNanos}, as soon as loop has finished and after {@link #statusNanos}
@@ -120,13 +120,13 @@ public DriverContext driverContext() {
      * Returns a blocked future when the chain of operators is blocked, allowing the caller
      * thread to do other work instead of blocking or busy-spinning on the blocked operator.
      */
-    private ListenableActionFuture<Void> run(TimeValue maxTime, int maxIterations) {
+    private SubscribableListener<Void> run(TimeValue maxTime, int maxIterations) {
         long maxTimeNanos = maxTime.nanos();
         long startTime = System.nanoTime();
         long nextStatus = startTime + statusNanos;
         int iter = 0;
         while (isFinished() == false) {
-            ListenableActionFuture<Void> fut = runSingleLoopIteration();
+            SubscribableListener<Void> fut = runSingleLoopIteration();
             if (fut.isDone() == false) {
                 status.set(updateStatus(DriverStatus.Status.ASYNC));
                 return fut;
@@ -166,7 +166,7 @@ public void close() {
         drainAndCloseOperators(null);
     }
 
-    private ListenableActionFuture<Void> runSingleLoopIteration() {
+    private SubscribableListener<Void> runSingleLoopIteration() {
         ensureNotCancelled();
         boolean movedPage = false;
 
@@ -227,7 +227,7 @@ private ListenableActionFuture<Void> runSingleLoopIteration() {
     public void cancel(String reason) {
         if (cancelReason.compareAndSet(null, reason)) {
             synchronized (this) {
-                ListenableActionFuture<Void> fut = this.blocked.get();
+                SubscribableListener<Void> fut = this.blocked.get();
                 if (fut != null) {
                     fut.onFailure(new TaskCancelledException(reason));
                 }
@@ -276,7 +276,7 @@ protected void doRun() {
                     listener.onResponse(null);
                     return;
                 }
-                ListenableActionFuture<Void> fut = driver.run(maxTime, maxIterations);
+                SubscribableListener<Void> fut = driver.run(maxTime, maxIterations);
                 if (fut.isDone()) {
                     schedule(maxTime, maxIterations, executor, driver, listener);
                 } else {
@@ -299,15 +299,15 @@ public void onFailure(Exception e) {
         });
     }
 
-    private static ListenableActionFuture<Void> oneOf(List<ListenableActionFuture<Void>> futures) {
+    private static SubscribableListener<Void> oneOf(List<SubscribableListener<Void>> futures) {
         if (futures.isEmpty()) {
             return Operator.NOT_BLOCKED;
         }
         if (futures.size() == 1) {
             return futures.get(0);
         }
-        ListenableActionFuture<Void> oneOf = new ListenableActionFuture<>();
-        for (ListenableActionFuture<Void> fut : futures) {
+        SubscribableListener<Void> oneOf = new SubscribableListener<>();
+        for (SubscribableListener<Void> fut : futures) {
             fut.addListener(oneOf);
         }
         return oneOf;
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Operator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Operator.java
index 1e61dc8010070..63dbdf2be09bf 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Operator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Operator.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.compute.operator;
 
-import org.elasticsearch.action.support.ListenableActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.compute.Describable;
@@ -88,17 +88,11 @@ default Status status() {
      * If the operator is not blocked, this method returns {@link #NOT_BLOCKED} which is an already
      * completed future.
      */
-    default ListenableActionFuture<Void> isBlocked() {
+    default SubscribableListener<Void> isBlocked() {
         return NOT_BLOCKED;
     }
 
-    ListenableActionFuture<Void> NOT_BLOCKED = newCompletedFuture();
-
-    static ListenableActionFuture<Void> newCompletedFuture() {
-        ListenableActionFuture<Void> fut = new ListenableActionFuture<>();
-        fut.onResponse(null);
-        return fut;
-    }
+    SubscribableListener<Void> NOT_BLOCKED = SubscribableListener.newSucceeded(null);
 
     /**
      * A factory for creating intermediate operators.
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java
index 5c2c8d4b79349..44759b0a430f4 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.compute.operator.exchange;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ListenableActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.Operator;
 
@@ -24,12 +24,12 @@ final class ExchangeBuffer {
     private final int maxSize;
 
     private final Object notEmptyLock = new Object();
-    private ListenableActionFuture<Void> notEmptyFuture = null;
+    private SubscribableListener<Void> notEmptyFuture = null;
 
     private final Object notFullLock = new Object();
-    private ListenableActionFuture<Void> notFullFuture = null;
+    private SubscribableListener<Void> notFullFuture = null;
 
-    private final ListenableActionFuture<Void> completionFuture = new ListenableActionFuture<>();
+    private final SubscribableListener<Void> completionFuture = new SubscribableListener<>();
 
     private volatile boolean noMoreInputs = false;
 
@@ -59,7 +59,7 @@ Page pollPage() {
     }
 
     private void notifyNotEmpty() {
-        final ListenableActionFuture<Void> toNotify;
+        final SubscribableListener<Void> toNotify;
         synchronized (notEmptyLock) {
             toNotify = notEmptyFuture;
             notEmptyFuture = null;
@@ -70,7 +70,7 @@ private void notifyNotEmpty() {
     }
 
     private void notifyNotFull() {
-        final ListenableActionFuture<Void> toNotify;
+        final SubscribableListener<Void> toNotify;
         synchronized (notFullLock) {
             toNotify = notFullFuture;
             notFullFuture = null;
@@ -80,7 +80,7 @@ private void notifyNotFull() {
         }
     }
 
-    ListenableActionFuture<Void> waitForWriting() {
+    SubscribableListener<Void> waitForWriting() {
         // maxBufferSize check is not water-tight as more than one sink can pass this check at the same time.
         if (queueSize.get() < maxSize || noMoreInputs) {
             return Operator.NOT_BLOCKED;
@@ -90,13 +90,13 @@ ListenableActionFuture<Void> waitForWriting() {
                 return Operator.NOT_BLOCKED;
             }
             if (notFullFuture == null) {
-                notFullFuture = new ListenableActionFuture<>();
+                notFullFuture = new SubscribableListener<>();
             }
             return notFullFuture;
         }
     }
 
-    ListenableActionFuture<Void> waitForReading() {
+    SubscribableListener<Void> waitForReading() {
         if (size() > 0 || noMoreInputs) {
             return Operator.NOT_BLOCKED;
         }
@@ -105,7 +105,7 @@ ListenableActionFuture<Void> waitForReading() {
                 return Operator.NOT_BLOCKED;
             }
             if (notEmptyFuture == null) {
-                notEmptyFuture = new ListenableActionFuture<>();
+                notEmptyFuture = new SubscribableListener<>();
             }
             return notEmptyFuture;
         }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSink.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSink.java
index 80a941711dfb5..8f0208740b689 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSink.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSink.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.compute.operator.exchange;
 
-import org.elasticsearch.action.support.ListenableActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.compute.data.Page;
 
 /**
@@ -33,5 +33,5 @@ public interface ExchangeSink {
     /**
      * Whether the sink is blocked on adding more pages
      */
-    ListenableActionFuture<Void> waitForWriting();
+    SubscribableListener<Void> waitForWriting();
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkHandler.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkHandler.java
index 901419f23ed56..9f921c40851d4 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkHandler.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkHandler.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.compute.operator.exchange;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ListenableActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.compute.data.Page;
 
 import java.util.Queue;
@@ -35,14 +35,13 @@ public final class ExchangeSinkHandler {
     // listeners are notified by only one thread.
     private final Semaphore promised = new Semaphore(1);
 
-    private final ListenableActionFuture<Void> completionFuture;
+    private final SubscribableListener<Void> completionFuture;
     private final LongSupplier nowInMillis;
     private final AtomicLong lastUpdatedInMillis;
 
     public ExchangeSinkHandler(int maxBufferSize, LongSupplier nowInMillis) {
         this.buffer = new ExchangeBuffer(maxBufferSize);
-        this.completionFuture = new ListenableActionFuture<>();
-        this.buffer.addCompletionListener(completionFuture);
+        this.completionFuture = SubscribableListener.newForked(buffer::addCompletionListener);
         this.nowInMillis = nowInMillis;
         this.lastUpdatedInMillis = new AtomicLong(nowInMillis.getAsLong());
     }
@@ -79,7 +78,7 @@ public boolean isFinished() {
         }
 
         @Override
-        public ListenableActionFuture<Void> waitForWriting() {
+        public SubscribableListener<Void> waitForWriting() {
             return buffer.waitForWriting();
         }
     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkOperator.java
index d1bbdd9e0afd2..d52f25e9d8306 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSinkOperator.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.compute.operator.exchange;
 
-import org.elasticsearch.action.support.ListenableActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -67,7 +67,7 @@ public void finish() {
     }
 
     @Override
-    public ListenableActionFuture<Void> isBlocked() {
+    public SubscribableListener<Void> isBlocked() {
         return sink.waitForWriting();
     }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSource.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSource.java
index 78249689e3bd8..01ed5e3fb6388 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSource.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSource.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.compute.operator.exchange;
 
-import org.elasticsearch.action.support.ListenableActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.compute.data.Page;
 
 /**
@@ -38,5 +38,5 @@ public interface ExchangeSource {
     /**
      * Allows callers to stop reading from the source when it's blocked
      */
-    ListenableActionFuture<Void> waitForReading();
+    SubscribableListener<Void> waitForReading();
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java
index d657e7a117dd1..b7dacdf2a6ea0 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceHandler.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ListenableActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.core.AbstractRefCounted;
@@ -34,7 +34,7 @@ public final class ExchangeSourceHandler extends AbstractRefCounted {
     private final PendingInstances outstandingSinks = new PendingInstances();
     private final PendingInstances outstandingSources = new PendingInstances();
     private final AtomicReference<Exception> failure = new AtomicReference<>();
-    private final ListenableActionFuture<Void> completionFuture = new ListenableActionFuture<>();
+    private final SubscribableListener<Void> completionFuture = new SubscribableListener<>();
 
     public ExchangeSourceHandler(int maxBufferSize, Executor fetchExecutor) {
         this.buffer = new ExchangeBuffer(maxBufferSize);
@@ -68,7 +68,7 @@ public boolean isFinished() {
         }
 
         @Override
-        public ListenableActionFuture<Void> waitForReading() {
+        public SubscribableListener<Void> waitForReading() {
             return buffer.waitForReading();
         }
 
@@ -164,7 +164,7 @@ void fetchPage() {
                     if (resp.finished()) {
                         onSinkComplete();
                     } else {
-                        ListenableActionFuture<Void> future = buffer.waitForWriting();
+                        SubscribableListener<Void> future = buffer.waitForWriting();
                         if (future.isDone()) {
                             if (loopControl.tryResume() == false) {
                                 fetchPage();
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceOperator.java
index e9726e91c73bf..8719ed6ab90ea 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeSourceOperator.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.compute.operator.exchange;
 
-import org.elasticsearch.action.support.ListenableActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -28,7 +28,7 @@
 public class ExchangeSourceOperator extends SourceOperator {
 
     private final ExchangeSource source;
-    private ListenableActionFuture<Void> isBlocked = NOT_BLOCKED;
+    private SubscribableListener<Void> isBlocked = NOT_BLOCKED;
     private int pagesEmitted;
 
     public record ExchangeSourceOperatorFactory(Supplier<ExchangeSource> exchangeSources) implements SourceOperatorFactory {
@@ -68,7 +68,7 @@ public void finish() {
     }
 
     @Override
-    public ListenableActionFuture<Void> isBlocked() {
+    public SubscribableListener<Void> isBlocked() {
         if (isBlocked.isDone()) {
             isBlocked = source.waitForReading();
             if (isBlocked.isDone()) {
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java
index 2c566aa46c413..7365a55778084 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java
@@ -10,8 +10,8 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.action.support.ListenableActionFuture;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.PageCacheRecycler;
@@ -148,7 +148,7 @@ public void close() {
         Page page1 = new Page(Block.constantNullBlock(1));
         operator.addInput(page1);
         assertFalse(operator.isBlocked().isDone());
-        ListenableActionFuture<Void> blocked1 = operator.isBlocked();
+        SubscribableListener<Void> blocked1 = operator.isBlocked();
         assertTrue(operator.needsInput());
 
         Page page2 = new Page(Block.constantNullBlock(2));
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java
index af6c89395f245..87fd1513aaf2d 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeServiceTests.java
@@ -10,8 +10,8 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ListenableActionFuture;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.node.VersionInformation;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -96,7 +96,7 @@ public void testBasic() throws Exception {
         assertThat(sourceExchanger.refCount(), equalTo(2));
         sourceExchanger.addRemoteSink(sinkExchanger::fetchPageAsync, 1);
         assertThat(sourceExchanger.refCount(), equalTo(3));
-        ListenableActionFuture<Void> waitForReading = source.waitForReading();
+        SubscribableListener<Void> waitForReading = source.waitForReading();
         assertFalse(waitForReading.isDone());
         assertNull(source.pollPage());
         assertTrue(sink1.waitForWriting().isDone());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
index e3d0034abde6b..5c14a19afd6f2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
@@ -16,9 +16,9 @@
 import org.elasticsearch.action.search.SearchShardsResponse;
 import org.elasticsearch.action.support.ChannelActionListener;
 import org.elasticsearch.action.support.ContextPreservingActionListener;
-import org.elasticsearch.action.support.ListenableActionFuture;
 import org.elasticsearch.action.support.RefCountingListener;
 import org.elasticsearch.action.support.RefCountingRunnable;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -186,7 +186,7 @@ private void runComputeOnRemoteNodes(
         Supplier<ActionListener<DataNodeResponse>> listener
     ) {
         // Do not complete the exchange sources until we have linked all remote sinks
-        final ListenableActionFuture<Void> blockingSinkFuture = new ListenableActionFuture<>();
+        final SubscribableListener<Void> blockingSinkFuture = new SubscribableListener<>();
         exchangeSource.addRemoteSink(
             (sourceFinished, l) -> blockingSinkFuture.addListener(l.map(ignored -> new ExchangeResponse(null, true))),
             1

From 1e9c165bf59a4ab4cc5f579d4603c8d5e4667883 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 14 Sep 2023 11:21:24 -0700
Subject: [PATCH 02/41] Add utility for loading SPI extensions (#99557)

A recurring pattern in extension classes is to include a static load
method. Extensions are mostly meant as a way for serverless to extend
functionality, and no other extensions are acceptable, so there is
either one or zero extensions found. This commit adds a utility class
for loading SPI extensions, which expects the singleton pattern.
BuildExtension and VersionExtension uses are changed to using the new
helper class.
---
 .../main/java/org/elasticsearch/Build.java    |  10 +-
 .../org/elasticsearch/TransportVersion.java   |   4 +-
 .../org/elasticsearch/index/IndexVersion.java |   4 +-
 .../internal/BuildExtension.java              |  16 ---
 .../internal/VersionExtension.java            |  16 ---
 .../plugins/ExtensionLoader.java              |  49 ++++++++
 .../plugins/ExtensionLoaderTests.java         | 107 ++++++++++++++++++
 .../test/PrivilegedOperations.java            |   7 ++
 8 files changed, 173 insertions(+), 40 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/plugins/ExtensionLoader.java
 create mode 100644 server/src/test/java/org/elasticsearch/plugins/ExtensionLoaderTests.java

diff --git a/server/src/main/java/org/elasticsearch/Build.java b/server/src/main/java/org/elasticsearch/Build.java
index acb61e5fb46b6..c34a9737bf799 100644
--- a/server/src/main/java/org/elasticsearch/Build.java
+++ b/server/src/main/java/org/elasticsearch/Build.java
@@ -14,10 +14,12 @@
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.internal.BuildExtension;
+import org.elasticsearch.plugins.ExtensionLoader;
 
 import java.io.IOException;
 import java.net.URL;
 import java.security.CodeSource;
+import java.util.ServiceLoader;
 import java.util.jar.JarInputStream;
 import java.util.jar.Manifest;
 
@@ -41,12 +43,8 @@ private static class CurrentHolder {
 
         // finds the pluggable current build, or uses the local build as a fallback
         private static Build findCurrent() {
-            var buildExtension = BuildExtension.load();
-            if (buildExtension == null) {
-                return findLocalBuild();
-            }
-            var build = buildExtension.getCurrentBuild();
-            return build;
+            var buildExtension = ExtensionLoader.loadSingleton(ServiceLoader.load(BuildExtension.class), () -> Build::findLocalBuild);
+            return buildExtension.getCurrentBuild();
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java
index efb3cca9a7c59..2aae6befb673a 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersion.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersion.java
@@ -12,8 +12,10 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.internal.VersionExtension;
+import org.elasticsearch.plugins.ExtensionLoader;
 
 import java.io.IOException;
+import java.util.ServiceLoader;
 
 /**
  * Represents the version of the wire protocol used to communicate between a pair of ES nodes.
@@ -109,7 +111,7 @@ private static class CurrentHolder {
 
         // finds the pluggable current version, or uses the given fallback
         private static TransportVersion findCurrent() {
-            var versionExtension = VersionExtension.load();
+            var versionExtension = ExtensionLoader.loadSingleton(ServiceLoader.load(VersionExtension.class), () -> null);
             if (versionExtension == null) {
                 return TransportVersions.LATEST_DEFINED;
             }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersion.java b/server/src/main/java/org/elasticsearch/index/IndexVersion.java
index 34f415e46462a..ef238d1fe87f0 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexVersion.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexVersion.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Assertions;
 import org.elasticsearch.internal.VersionExtension;
+import org.elasticsearch.plugins.ExtensionLoader;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -25,6 +26,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.NavigableMap;
+import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.TreeMap;
 
@@ -154,7 +156,7 @@ private static class CurrentHolder {
 
         // finds the pluggable current version, or uses the given fallback
         private static IndexVersion findCurrent(IndexVersion fallback) {
-            var versionExtension = VersionExtension.load();
+            var versionExtension = ExtensionLoader.loadSingleton(ServiceLoader.load(VersionExtension.class), () -> null);
             if (versionExtension == null) {
                 return fallback;
             }
diff --git a/server/src/main/java/org/elasticsearch/internal/BuildExtension.java b/server/src/main/java/org/elasticsearch/internal/BuildExtension.java
index 8826045b93e38..0d1a9880f8d32 100644
--- a/server/src/main/java/org/elasticsearch/internal/BuildExtension.java
+++ b/server/src/main/java/org/elasticsearch/internal/BuildExtension.java
@@ -10,8 +10,6 @@
 
 import org.elasticsearch.Build;
 
-import java.util.ServiceLoader;
-
 /**
  * Allows plugging in current build info.
  */
@@ -21,18 +19,4 @@ public interface BuildExtension {
      * Returns the {@link Build} that represents the running Elasticsearch code.
      */
     Build getCurrentBuild();
-
-    /**
-     * Loads a single BuildExtension, or returns {@code null} if none are found.
-     */
-    static BuildExtension load() {
-        var loader = ServiceLoader.load(BuildExtension.class);
-        var extensions = loader.stream().toList();
-        if (extensions.size() > 1) {
-            throw new IllegalStateException("More than one build extension found");
-        } else if (extensions.size() == 0) {
-            return null;
-        }
-        return extensions.get(0).get();
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/internal/VersionExtension.java b/server/src/main/java/org/elasticsearch/internal/VersionExtension.java
index 8afef81455a95..83974b3b65158 100644
--- a/server/src/main/java/org/elasticsearch/internal/VersionExtension.java
+++ b/server/src/main/java/org/elasticsearch/internal/VersionExtension.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.index.IndexVersion;
 
-import java.util.ServiceLoader;
-
 /**
  * Allows plugging in current version elements.
  */
@@ -30,18 +28,4 @@ public interface VersionExtension {
      * This must be at least equal to the latest version found in {@link IndexVersion} V_* constants.
      */
     IndexVersion getCurrentIndexVersion();
-
-    /**
-     * Loads a single VersionExtension, or returns {@code null} if none are found.
-     */
-    static VersionExtension load() {
-        var loader = ServiceLoader.load(VersionExtension.class);
-        var extensions = loader.stream().toList();
-        if (extensions.size() > 1) {
-            throw new IllegalStateException("More than one version extension found");
-        } else if (extensions.size() == 0) {
-            return null;
-        }
-        return extensions.get(0).get();
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/plugins/ExtensionLoader.java b/server/src/main/java/org/elasticsearch/plugins/ExtensionLoader.java
new file mode 100644
index 0000000000000..7dfb64c989ea2
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/plugins/ExtensionLoader.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins;
+
+import java.util.Locale;
+import java.util.ServiceLoader;
+import java.util.function.Supplier;
+
+/**
+ * A utility for loading SPI extensions.
+ */
+public class ExtensionLoader {
+
+    /**
+     * Loads a single SPI extension.
+     *
+     * There should be no more than one extension found. If no service providers
+     * are found, the supplied fallback is used.
+     *
+     * Note: A ServiceLoader is needed rather than the service class because ServiceLoaders
+     * must be loaded by a module with the {@code uses} declaration. Since this
+     * utility class is in server, it will not have uses (or even know about) all the
+     * service classes it may load. Thus, the caller must load the ServiceLoader.
+     *
+     * @param loader a service loader instance to find the singleton extension in
+     * @param fallback a supplier for an instance if no extensions are found
+     * @return an instance of the extension
+     * @param <T> the SPI extension type
+     */
+    public static <T> T loadSingleton(ServiceLoader<T> loader, Supplier<T> fallback) {
+        var extensions = loader.stream().toList();
+        if (extensions.size() > 1) {
+            // It would be really nice to give the actual extension class here directly, but that would require passing it
+            // in effectively twice in the call site, once to ServiceLoader, and then to this method directly as well.
+            // It's annoying that ServiceLoader hangs onto the service class, but does not expose it. It does at least
+            // print the service class from its toString, which is better tha nothing
+            throw new IllegalStateException(String.format(Locale.ROOT, "More than one extension found for %s", loader));
+        } else if (extensions.isEmpty()) {
+            return fallback.get();
+        }
+        return extensions.get(0).get();
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/plugins/ExtensionLoaderTests.java b/server/src/test/java/org/elasticsearch/plugins/ExtensionLoaderTests.java
new file mode 100644
index 0000000000000..35f65ebedf9b9
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/plugins/ExtensionLoaderTests.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.PrivilegedOperations.ClosableURLClassLoader;
+import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
+import org.elasticsearch.test.jar.JarUtils;
+
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.ServiceLoader;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+
+public class ExtensionLoaderTests extends ESTestCase {
+    public interface TestService {
+        int getValue();
+    }
+
+    private ClosableURLClassLoader buildProviderJar(Map<String, CharSequence> sources) throws Exception {
+        var classToBytes = InMemoryJavaCompiler.compile(sources);
+
+        Map<String, byte[]> jarEntries = new HashMap<>();
+        for (var entry : sources.entrySet()) {
+            var classname = entry.getKey();
+            var filename = classname.replace(".", "/") + ".class";
+            jarEntries.put(filename, classToBytes.get(classname));
+        }
+        String serviceFile = String.join("\n", sources.keySet());
+        jarEntries.put(
+            "META-INF/services/org.elasticsearch.plugins.ExtensionLoaderTests$TestService",
+            serviceFile.getBytes(StandardCharsets.UTF_8)
+        );
+
+        Path topLevelDir = createTempDir(getTestName());
+        Path jar = topLevelDir.resolve("provider.jar");
+        JarUtils.createJarWithEntries(jar, jarEntries);
+        URL[] urls = new URL[] { jar.toUri().toURL() };
+
+        return new ClosableURLClassLoader(URLClassLoader.newInstance(urls, this.getClass().getClassLoader()));
+    }
+
+    private String defineProvider(String name, int value) {
+        return String.format(Locale.ROOT, """
+            package p;
+            import org.elasticsearch.plugins.ExtensionLoaderTests.TestService;
+            public class %s implements TestService {
+                @Override
+                public int getValue() {
+                    return %d;
+                }
+            }
+            """, name, value);
+    }
+
+    public void testNoProviderNullFallback() {
+        TestService service = ExtensionLoader.loadSingleton(ServiceLoader.load(TestService.class), () -> null);
+        assertThat(service, nullValue());
+    }
+
+    public void testNoProvider() {
+        TestService service = ExtensionLoader.loadSingleton(ServiceLoader.load(TestService.class), () -> () -> 2);
+        assertThat(service, not(nullValue()));
+        assertThat(service.getValue(), equalTo(2));
+    }
+
+    public void testOneProvider() throws Exception {
+        Map<String, CharSequence> sources = Map.of("p.FooService", defineProvider("FooService", 1));
+        try (var loader = buildProviderJar(sources)) {
+            TestService service = ExtensionLoader.loadSingleton(ServiceLoader.load(TestService.class, loader.classloader()), () -> null);
+            assertThat(service, not(nullValue()));
+            assertThat(service.getValue(), equalTo(1));
+        }
+    }
+
+    public void testManyProviders() throws Exception {
+        Map<String, CharSequence> sources = Map.of(
+            "p.FooService",
+            defineProvider("FooService", 1),
+            "p.BarService",
+            defineProvider("BarService", 2)
+        );
+        try (var loader = buildProviderJar(sources)) {
+            var e = expectThrows(
+                IllegalStateException.class,
+                () -> ExtensionLoader.loadSingleton(ServiceLoader.load(TestService.class, loader.classloader()), () -> null)
+            );
+            assertThat(e.getMessage(), containsString("More than one extension found"));
+            assertThat(e.getMessage(), containsString("TestService"));
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/PrivilegedOperations.java b/test/framework/src/main/java/org/elasticsearch/test/PrivilegedOperations.java
index 3c0a84b15a55e..ae819a57173cf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/PrivilegedOperations.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/PrivilegedOperations.java
@@ -53,6 +53,13 @@ public static void closeURLClassLoader(URLClassLoader loader) throws IOException
         }
     }
 
+    public record ClosableURLClassLoader(URLClassLoader classloader) implements AutoCloseable {
+        @Override
+        public void close() throws Exception {
+            closeURLClassLoader(classloader);
+        }
+    }
+
     public static Boolean compilationTaskCall(JavaCompiler.CompilationTask compilationTask) {
         return AccessController.doPrivileged(
             (PrivilegedAction<Boolean>) () -> compilationTask.call(),

From 9a8f13fd3be12b8e05591b9e8fac65a980a1f4eb Mon Sep 17 00:00:00 2001
From: Brian Seeders <brian.seeders@elastic.co>
Date: Thu, 14 Sep 2023 15:29:31 -0400
Subject: [PATCH 03/41] [buildkite] Add more memory to platform-support job
 agents (#99594)

---
 .buildkite/pipelines/periodic-platform-support.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml
index 520089286ec36..8522ead742768 100644
--- a/.buildkite/pipelines/periodic-platform-support.yml
+++ b/.buildkite/pipelines/periodic-platform-support.yml
@@ -27,7 +27,7 @@ steps:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}
           diskSizeGb: 350
-          machineType: custom-32-98304
+          machineType: n1-standard-32
         env: {}
   - group: platform-support-windows
     steps:
@@ -50,7 +50,7 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           diskType: pd-ssd
           diskSizeGb: 350
         env:

From a42145957fecc45a15297d296ed6298723f54038 Mon Sep 17 00:00:00 2001
From: Brian Seeders <brian.seeders@elastic.co>
Date: Thu, 14 Sep 2023 16:33:46 -0400
Subject: [PATCH 04/41] [buildkite] Disable all jenkins PR jobs when
 buildkite-opt-in label is present (#99597)

---
 ...elastic+elasticsearch+pull-request+build-benchmark-part1.yml | 2 ++
 ...elastic+elasticsearch+pull-request+build-benchmark-part2.yml | 2 ++
 ...elastic+elasticsearch+pull-request+bwc-snapshots-windows.yml | 1 +
 .ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots.yml | 1 +
 .ci/jobs.t/elastic+elasticsearch+pull-request+cloud-deploy.yml  | 2 ++
 .ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml    | 1 +
 .../elastic+elasticsearch+pull-request+eql-correctness.yml      | 1 +
 .../elastic+elasticsearch+pull-request+example-plugins.yml      | 2 ++
 .ci/jobs.t/elastic+elasticsearch+pull-request+full-bwc.yml      | 1 +
 ...c+elasticsearch+pull-request+packaging-tests-unix-sample.yml | 1 +
 .../elastic+elasticsearch+pull-request+packaging-tests-unix.yml | 1 +
 ...elasticsearch+pull-request+packaging-tests-windows-nojdk.yml | 1 +
 ...search+pull-request+packaging-tests-windows-sample-nojdk.yml | 1 +
 ...lasticsearch+pull-request+packaging-tests-windows-sample.yml | 1 +
 ...astic+elasticsearch+pull-request+packaging-tests-windows.yml | 1 +
 ...astic+elasticsearch+pull-request+packaging-upgrade-tests.yml | 1 +
 .ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml   | 1 +
 .../elastic+elasticsearch+pull-request+part-1-windows.yml       | 1 +
 .ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml   | 1 +
 .../elastic+elasticsearch+pull-request+part-2-windows.yml       | 1 +
 .ci/jobs.t/elastic+elasticsearch+pull-request+part-3-fips.yml   | 1 +
 .../elastic+elasticsearch+pull-request+part-3-windows.yml       | 1 +
 .ci/jobs.t/elastic+elasticsearch+pull-request+part-3.yml        | 1 +
 .ci/jobs.t/elastic+elasticsearch+pull-request+precommit.yml     | 2 ++
 .ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml | 2 ++
 .../elastic+elasticsearch+pull-request+rest-compatibility.yml   | 1 +
 .ci/templates.t/pull-request-gradle-unix.yml                    | 1 +
 27 files changed, 33 insertions(+)

diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+build-benchmark-part1.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+build-benchmark-part1.yml
index 064ac3022141f..a3f1345a07f13 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+build-benchmark-part1.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+build-benchmark-part1.yml
@@ -34,6 +34,8 @@
             - ^x-pack/docs/.*
           white-list-labels:
             - 'build-benchmark'
+          black-list-labels:
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+build-benchmark-part2.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+build-benchmark-part2.yml
index bf72ffa54cdae..f1b11ab1ec75a 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+build-benchmark-part2.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+build-benchmark-part2.yml
@@ -34,6 +34,8 @@
             - ^x-pack/docs/.*
           white-list-labels:
             - 'build-benchmark'
+          black-list-labels:
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots-windows.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots-windows.yml
index 629700daa70e2..c0ed9bf998159 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots-windows.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots-windows.yml
@@ -29,6 +29,7 @@
             - 'test-windows'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     axes:
       - axis:
           type: slave
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots.yml
index 86da8986f27e5..676f5f6f629b7 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots.yml
@@ -26,6 +26,7 @@
           black-list-labels:
             - '>test-mute'
             - 'test-full-bwc'
+            - 'buildkite-opt-in'
     axes:
       - axis:
           type: slave
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+cloud-deploy.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+cloud-deploy.yml
index eedbf7bba5789..24548954d8a10 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+cloud-deploy.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+cloud-deploy.yml
@@ -26,6 +26,8 @@
             - ^x-pack/docs/.*
           white-list-labels:
             - 'cloud-deploy'
+          black-list-labels:
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml
index 77c499f455b22..c766b4379a1f6 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml
@@ -23,6 +23,7 @@
             - ^x-pack/docs/.*
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+eql-correctness.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+eql-correctness.yml
index fcdbf2ea87084..0b9eea62ad9bf 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+eql-correctness.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+eql-correctness.yml
@@ -25,6 +25,7 @@
             - ^x-pack/docs/.*
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+example-plugins.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+example-plugins.yml
index f79c98c00101f..320a9c6176d5f 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+example-plugins.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+example-plugins.yml
@@ -23,6 +23,8 @@
             - build-tools/.*
             - build-tools-internal/.*
             - plugins/examples/.*
+          black-list-labels:
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+full-bwc.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+full-bwc.yml
index 5276d39f956d3..2a7920e4bae89 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+full-bwc.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+full-bwc.yml
@@ -27,6 +27,7 @@
             - 'test-full-bwc'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     axes:
       - axis:
           type: slave
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml
index c283da8e32479..04e48036a8e9e 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml
@@ -27,6 +27,7 @@
           black-list-labels:
             - '>test-mute'
             - ':Delivery/Packaging'
+            - 'buildkite-opt-in'
     axes:
       - axis:
           type: label-expression
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix.yml
index 95a4c4273ebb7..a7413699ff6c3 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix.yml
@@ -28,6 +28,7 @@
             - ':Delivery/Packaging'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     axes:
       - axis:
           type: label-expression
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-nojdk.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-nojdk.yml
index ecd4a1a084755..ea4097b1a0b93 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-nojdk.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-nojdk.yml
@@ -32,6 +32,7 @@
             - ':Delivery/Packaging'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     axes:
       - axis:
           type: label-expression
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample-nojdk.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample-nojdk.yml
index 091dcf9eb77a0..ec644445ef8de 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample-nojdk.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample-nojdk.yml
@@ -31,6 +31,7 @@
           black-list-labels:
             - '>test-mute'
             - ':Delivery/Packaging'
+            - 'buildkite-opt-in'
     axes:
       - axis:
           type: label-expression
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml
index a438335529a3c..242e137cb1d83 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml
@@ -30,6 +30,7 @@
           black-list-labels:
             - '>test-mute'
             - ':Delivery/Packaging'
+            - 'buildkite-opt-in'
     axes:
       - axis:
           type: label-expression
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows.yml
index 1dc0127284c47..a2ffc7b4050ec 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows.yml
@@ -32,6 +32,7 @@
             - ':Delivery/Packaging'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     axes:
       - axis:
           type: label-expression
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-upgrade-tests.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-upgrade-tests.yml
index 01770b7d94c6e..2b73d0144cab7 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-upgrade-tests.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-upgrade-tests.yml
@@ -29,6 +29,7 @@
             - ':Delivery/Packaging'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     axes:
       - axis:
           type: label-expression
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml
index 793ade87a1fd9..a661230d3b93b 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml
@@ -27,6 +27,7 @@
             - 'Team:Security'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     builders:
       - inject:
           # Use FIPS-specific Java versions
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-windows.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-windows.yml
index 9a55d8dc6eeac..d7afdd0ac3277 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-windows.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-windows.yml
@@ -28,6 +28,7 @@
             - 'test-windows'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml
index 0795172b916e2..913820709dabc 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml
@@ -27,6 +27,7 @@
             - 'Team:Security'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     builders:
       - inject:
           # Use FIPS-specific Java versions
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-windows.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-windows.yml
index de09d5044d466..ae590872be16e 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-windows.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-windows.yml
@@ -28,6 +28,7 @@
             - 'test-windows'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-fips.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-fips.yml
index 3383e81ae61ed..6bf6544d40310 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-fips.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-fips.yml
@@ -28,6 +28,7 @@
             - 'Team:Security'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     builders:
       - inject:
           # Use FIPS-specific Java versions
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-windows.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-windows.yml
index 65ed5c58335a9..58bad17954b24 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-windows.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-windows.yml
@@ -29,6 +29,7 @@
             - 'test-windows'
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3.yml
index 325b9ecb68fd9..0158b909903b4 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3.yml
@@ -23,6 +23,7 @@
             - ^x-pack/docs/.*
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
           black-list-target-branches:
             - 6.8
             - 7.17
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+precommit.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+precommit.yml
index 60878f1519555..1267b6a21778e 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+precommit.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+precommit.yml
@@ -20,6 +20,8 @@
           cancel-builds-on-update: true
           white-list-labels:
             - '>test-mute'
+          black-list-labels:
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml
index 98ada6e2080e4..1ab6bd1ce0e5d 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml
@@ -25,6 +25,8 @@
             - ^x-pack/docs/.*
           white-list-labels:
             - 'test-release'
+          black-list-labels:
+            - 'buildkite-opt-in'
           black-list-target-branches:
             - 7.15
             - 6.8
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+rest-compatibility.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+rest-compatibility.yml
index 417ce525880d6..216f8ceae2078 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+rest-compatibility.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+rest-compatibility.yml
@@ -28,6 +28,7 @@
             - ^x-pack/docs/.*
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
diff --git a/.ci/templates.t/pull-request-gradle-unix.yml b/.ci/templates.t/pull-request-gradle-unix.yml
index b4b5c48739097..c09e64c56f32d 100644
--- a/.ci/templates.t/pull-request-gradle-unix.yml
+++ b/.ci/templates.t/pull-request-gradle-unix.yml
@@ -23,6 +23,7 @@
             - ^x-pack/docs/.*
           black-list-labels:
             - '>test-mute'
+            - 'buildkite-opt-in'
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

From b1e69ff21bc7f92dd6f14175ac05dd6cb6a67b18 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Thu, 14 Sep 2023 14:13:00 -0700
Subject: [PATCH 05/41] AwaitFix EsqlActionTaskIT

Relates #99589
---
 .../org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java
index 92b81057ec698..ff6a7b15c3462 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.action;
 
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction;
@@ -65,6 +66,7 @@
     value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.tasks.TaskCancellationService:TRACE",
     reason = "These tests are failing frequently; we need logs before muting them"
 )
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99589")
 public class EsqlActionTaskIT extends AbstractEsqlIntegTestCase {
     private static int PAGE_SIZE;
     private static int NUM_DOCS;

From bba26ae388a706c0de0458f43ae6cea9fbef6540 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 14 Sep 2023 14:32:28 -0700
Subject: [PATCH 06/41] Add rest filtering to RestExtension (#99559)

This commit modifies the recently added RestExtension so that it can
also filter rest handlers. The construction of the extension must change
slightly so that the extension may bind to elements from a plugin (eg
ThreadPool to get ThreadContext) which it would not have in a pure
static environment of Java's SPI.
---
 .../elasticsearch/action/ActionModule.java    | 24 ++++---------
 .../java/org/elasticsearch/node/Node.java     |  4 ++-
 .../elasticsearch/plugins/PluginsService.java | 22 ++++++++++++
 .../plugins/internal/RestExtension.java       | 36 +++++++++++++------
 .../action/ActionModuleTests.java             | 16 ++++++---
 .../AbstractHttpServerTransportTests.java     |  4 ++-
 .../xpack/security/SecurityTests.java         |  4 ++-
 7 files changed, 73 insertions(+), 37 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 5683ace63ba3b..31355aac21d67 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -510,6 +510,7 @@ public class ActionModule extends AbstractModule {
     private final ThreadPool threadPool;
     private final ReservedClusterStateService reservedClusterStateService;
     private final boolean serverlessEnabled;
+    private final RestExtension restExtension;
 
     public ActionModule(
         Settings settings,
@@ -525,7 +526,8 @@ public ActionModule(
         SystemIndices systemIndices,
         Tracer tracer,
         ClusterService clusterService,
-        List<ReservedClusterStateHandler<?>> reservedStateHandlers
+        List<ReservedClusterStateHandler<?>> reservedStateHandlers,
+        RestExtension restExtension
     ) {
         this.settings = settings;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
@@ -572,6 +574,7 @@ public ActionModule(
             restController = new RestController(restInterceptor, nodeClient, circuitBreakerService, usageService, tracer);
         }
         reservedClusterStateService = new ReservedClusterStateService(clusterService, reservedStateHandlers);
+        this.restExtension = restExtension;
     }
 
     private static <T> T getRestServerComponent(
@@ -851,15 +854,10 @@ private static ActionFilters setupActionFilters(List<ActionPlugin> actionPlugins
 
     public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         List<AbstractCatAction> catActions = new ArrayList<>();
-        var restExtension = RestExtension.load(() -> new RestExtension() {
-            @Override
-            public Predicate<AbstractCatAction> getCatActionsFilter() {
-                return action -> true;
-            }
-        });
         Predicate<AbstractCatAction> catActionsFilter = restExtension.getCatActionsFilter();
+        Predicate<RestHandler> restFilter = restExtension.getActionsFilter();
         Consumer<RestHandler> registerHandler = handler -> {
-            if (shouldKeepRestHandler(handler)) {
+            if (restFilter.test(handler)) {
                 if (handler instanceof AbstractCatAction catAction && catActionsFilter.test(catAction)) {
                     catActions.add(catAction);
                 }
@@ -1066,16 +1064,6 @@ public Predicate<AbstractCatAction> getCatActionsFilter() {
         registerHandler.accept(new RestDeleteSynonymRuleAction());
     }
 
-    /**
-     * This method is used to determine whether a RestHandler ought to be kept in memory or not. Returns true if serverless mode is
-     * disabled, or if there is any ServlerlessScope annotation on the RestHandler.
-     * @param handler
-     * @return
-     */
-    private boolean shouldKeepRestHandler(final RestHandler handler) {
-        return serverlessEnabled == false || handler.getServerlessScope() != null;
-    }
-
     @Override
     protected void configure() {
         bind(ActionFilters.class).toInstance(actionFilters);
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index 111dc5ec72165..1ae3aaa9e09db 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -183,6 +183,7 @@
 import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.plugins.internal.DocumentParsingObserverPlugin;
 import org.elasticsearch.plugins.internal.ReloadAwarePlugin;
+import org.elasticsearch.plugins.internal.RestExtension;
 import org.elasticsearch.plugins.internal.SettingsExtension;
 import org.elasticsearch.readiness.ReadinessService;
 import org.elasticsearch.repositories.RepositoriesModule;
@@ -814,7 +815,8 @@ protected Node(
                 systemIndices,
                 tracer,
                 clusterService,
-                reservedStateHandlers
+                reservedStateHandlers,
+                pluginsService.loadSingletonServiceProvider(RestExtension.class, RestExtension::allowAll)
             );
             modules.add(actionModule);
 
diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
index be8f9b303f4eb..96f3eedde165c 100644
--- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -59,6 +59,7 @@
 import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -345,6 +346,27 @@ public <T> List<? extends T> loadServiceProviders(Class<T> service) {
         return Collections.unmodifiableList(result);
     }
 
+    /**
+     * Loads a single SPI extension.
+     *
+     * There should be no more than one extension found. If no service providers
+     * are found, the supplied fallback is used.
+     *
+     * @param service the SPI class that should be loaded
+     * @param fallback a supplier for an instance if no providers are found
+     * @return an instance of the service
+     * @param <T> the SPI service type
+     */
+    public <T> T loadSingletonServiceProvider(Class<T> service, Supplier<T> fallback) {
+        var services = loadServiceProviders(service);
+        if (services.size() > 1) {
+            throw new IllegalStateException(String.format(Locale.ROOT, "More than one extension found for %s", service.getSimpleName()));
+        } else if (services.size() == 0) {
+            return fallback.get();
+        }
+        return services.get(0);
+    }
+
     private static void loadExtensionsForPlugin(ExtensiblePlugin extensiblePlugin, List<Plugin> extendingPlugins) {
         ExtensiblePlugin.ExtensionLoader extensionLoader = new ExtensiblePlugin.ExtensionLoader() {
             @Override
diff --git a/server/src/main/java/org/elasticsearch/plugins/internal/RestExtension.java b/server/src/main/java/org/elasticsearch/plugins/internal/RestExtension.java
index da5de4f784a22..4864e6bf31222 100644
--- a/server/src/main/java/org/elasticsearch/plugins/internal/RestExtension.java
+++ b/server/src/main/java/org/elasticsearch/plugins/internal/RestExtension.java
@@ -8,11 +8,10 @@
 
 package org.elasticsearch.plugins.internal;
 
+import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.rest.action.cat.AbstractCatAction;
 
-import java.util.ServiceLoader;
 import java.util.function.Predicate;
-import java.util.function.Supplier;
 
 public interface RestExtension {
     /**
@@ -23,14 +22,29 @@ public interface RestExtension {
      */
     Predicate<AbstractCatAction> getCatActionsFilter();
 
-    static RestExtension load(Supplier<RestExtension> fallback) {
-        var loader = ServiceLoader.load(RestExtension.class);
-        var extensions = loader.stream().toList();
-        if (extensions.size() > 1) {
-            throw new IllegalStateException("More than one rest extension found");
-        } else if (extensions.size() == 0) {
-            return fallback.get();
-        }
-        return extensions.get(0).get();
+    /**
+     * Returns a filter that determines which rest actions are exposed.
+     *
+     * The filter should return {@code false} if an action should be included,
+     * or {@code false} if the paths
+     * @return
+     */
+    Predicate<RestHandler> getActionsFilter();
+
+    /**
+     * Returns a rest extension which allows all rest endpoints through.
+     */
+    static RestExtension allowAll() {
+        return new RestExtension() {
+            @Override
+            public Predicate<AbstractCatAction> getCatActionsFilter() {
+                return action -> true;
+            }
+
+            @Override
+            public Predicate<RestHandler> getActionsFilter() {
+                return handler -> true;
+            }
+        };
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
index 6975cc22adaa9..1279ea810f0a6 100644
--- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ActionPlugin.ActionHandler;
 import org.elasticsearch.plugins.interceptor.RestServerActionPlugin;
+import org.elasticsearch.plugins.internal.RestExtension;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
@@ -123,7 +124,8 @@ public void testSetupRestHandlerContainsKnownBuiltin() {
             null,
             null,
             mock(ClusterService.class),
-            List.of()
+            List.of(),
+            RestExtension.allowAll()
         );
         actionModule.initRestHandlers(null);
         // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
@@ -182,7 +184,8 @@ public String getName() {
                 null,
                 null,
                 mock(ClusterService.class),
-                List.of()
+                List.of(),
+                RestExtension.allowAll()
             );
             Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null));
             assertThat(e.getMessage(), startsWith("Cannot replace existing handler for [/_nodes] for method: GET"));
@@ -234,7 +237,8 @@ public List<RestHandler> getRestHandlers(
                 null,
                 null,
                 mock(ClusterService.class),
-                List.of()
+                List.of(),
+                RestExtension.allowAll()
             );
             actionModule.initRestHandlers(null);
             // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
@@ -281,7 +285,8 @@ public void test3rdPartyHandlerIsNotInstalled() {
                     null,
                     null,
                     mock(ClusterService.class),
-                    List.of()
+                    List.of(),
+                    RestExtension.allowAll()
                 )
             );
             assertThat(
@@ -319,7 +324,8 @@ public void test3rdPartyRestControllerIsNotInstalled() {
                     null,
                     null,
                     mock(ClusterService.class),
-                    List.of()
+                    List.of(),
+                    RestExtension.allowAll()
                 )
             );
             assertThat(
diff --git a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java
index e32fa310ec5c8..b8d102db7e8ae 100644
--- a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java
+++ b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.indices.TestIndexNameExpressionResolver;
 import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.internal.RestExtension;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestControllerTests;
 import org.elasticsearch.rest.RestHeaderDefinition;
@@ -1152,7 +1153,8 @@ public Collection<RestHeaderDefinition> getRestHeaders() {
             null,
             null,
             mock(ClusterService.class),
-            List.of()
+            List.of(),
+            RestExtension.allowAll()
         );
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
index 72f3c2be7087f..fa7ff7b12994c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
@@ -48,6 +48,7 @@
 import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.internal.DocumentParsingObserver;
+import org.elasticsearch.plugins.internal.RestExtension;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -775,7 +776,8 @@ public void testSecurityRestHandlerInterceptorCanBeInstalled() throws IllegalAcc
                 null,
                 Tracer.NOOP,
                 mock(ClusterService.class),
-                List.of()
+                List.of(),
+                RestExtension.allowAll()
             );
             actionModule.initRestHandlers(null);
 

From 460c2ee0f6e6f915f3266994206c894ee05d73d9 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Thu, 14 Sep 2023 14:58:33 -0700
Subject: [PATCH 07/41] Avoid using query pragmas in ESQL tests (#99600)

This PR avoids enabling QueryPragmas in ESQL tests since it's only
available in the snapshot build.
---
 .../xpack/esql/action/AbstractEsqlIntegTestCase.java   |  6 ++++--
 .../elasticsearch/xpack/esql/action/EsqlActionIT.java  |  3 +--
 .../xpack/esql/action/EsqlActionTaskIT.java            |  1 +
 .../elasticsearch/xpack/esql/action/ManyShardsIT.java  | 10 +++++-----
 4 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java
index 9598d8d4498cb..2c904a39129db 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java
@@ -85,8 +85,7 @@ protected EsqlQueryResponse run(EsqlQueryRequest request) {
 
     protected static QueryPragmas randomPragmas() {
         Settings.Builder settings = Settings.builder();
-        // pragmas are only enabled on snapshot builds
-        if (Build.current().isSnapshot()) {
+        if (canUseQueryPragmas()) {
             if (randomBoolean()) {
                 settings.put("task_concurrency", randomLongBetween(1, 10));
             }
@@ -118,4 +117,7 @@ protected static QueryPragmas randomPragmas() {
         return new QueryPragmas(settings.build());
     }
 
+    protected static boolean canUseQueryPragmas() {
+        return Build.current().isSnapshot();
+    }
 }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java
index c567edda97018..fdc83161dcd06 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java
@@ -445,8 +445,7 @@ public void testFromEvalStats() {
         assertEquals(0.034d, (double) getValuesList(results).get(0).get(0), 0.001d);
     }
 
-    public void testFromStatsEvalWithPragma() {
-        assumeTrue("pragmas only enabled on snapshot builds", Build.current().isSnapshot());
+    public void testFromStatsThenEval() {
         EsqlQueryResponse results = run("from test | stats avg_count = avg(count) | eval x = avg_count + 7");
         logger.info(results);
         Assert.assertEquals(1, getValuesList(results).size());
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java
index ff6a7b15c3462..16f704aa8f7c3 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java
@@ -82,6 +82,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
 
     @Before
     public void setupIndex() throws IOException {
+        assumeTrue("requires query pragmas", canUseQueryPragmas());
         PAGE_SIZE = between(10, 100);
         NUM_DOCS = between(4 * PAGE_SIZE, 5 * PAGE_SIZE);
         READ_DESCRIPTION = """
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java
index a95f601d88ca0..c1476c8c52de5 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java
@@ -59,11 +59,11 @@ public void testConcurrentQueries() throws Exception {
                 } catch (InterruptedException e) {
                     throw new AssertionError(e);
                 }
-                var pragmas = Settings.builder()
-                    .put(randomPragmas().getSettings())
-                    .put("exchange_concurrent_clients", between(1, 2))
-                    .build();
-                run("from test-* | stats count(user) by tags", new QueryPragmas(pragmas));
+                final var pragmas = Settings.builder();
+                if (canUseQueryPragmas()) {
+                    pragmas.put(randomPragmas().getSettings()).put("exchange_concurrent_clients", between(1, 2));
+                }
+                run("from test-* | stats count(user) by tags", new QueryPragmas(pragmas.build()));
             });
         }
         for (Thread thread : threads) {

From 89d11c66ebbe4959ee6a5a6ec9fb9b5435b9d58a Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 14 Sep 2023 21:58:27 -0700
Subject: [PATCH 08/41] Do not run die-with-dignity test on release builds
 (#99440)

The test external modules only exist in snapshot builds. This commit
fixes the module test to only run on snapshot builds.

fixes #99402
---
 test/external-modules/die-with-dignity/build.gradle | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/external-modules/die-with-dignity/build.gradle b/test/external-modules/die-with-dignity/build.gradle
index 93582c9decb86..999b81af027b3 100644
--- a/test/external-modules/die-with-dignity/build.gradle
+++ b/test/external-modules/die-with-dignity/build.gradle
@@ -19,3 +19,7 @@ tasks.named("test").configure {
 tasks.named("yamlRestTest").configure {
   enabled = false
 }
+
+tasks.named('javaRestTest').configure {
+  it.onlyIf("snapshot build") { BuildParams.isSnapshotBuild() }
+}

From cb42b964e53f725bb41ac93108d5afe8abe7be7a Mon Sep 17 00:00:00 2001
From: Andrei Dan <andrei.dan@elastic.co>
Date: Fri, 15 Sep 2023 11:06:42 +0100
Subject: [PATCH 09/41] [DSL] Add fixed_interval min value validation for a
 downsampling round (#99544)

This adds validationg for the downsampling round to have a minimum
`fixed_interval` of `5m`.
---
 .../DataStreamLifecycleServiceTests.java      |  8 +++---
 .../cluster/metadata/DataStreamLifecycle.java | 10 +++++++
 .../metadata/DataStreamLifecycleTests.java    | 18 ++++++++++++
 .../cluster/metadata/DataStreamTests.java     | 12 ++++----
 ...StreamLifecycleDownsampleDisruptionIT.java |  4 +--
 .../DataStreamLifecycleDownsampleIT.java      | 28 +++++++++----------
 ...StreamLifecycleDownsamplingSecurityIT.java | 16 +++++------
 7 files changed, 62 insertions(+), 34 deletions(-)

diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
index 3a5afd2042565..5b73d94be578a 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
@@ -923,7 +923,7 @@ public void testDownsampling() throws Exception {
             DataStreamLifecycle.newBuilder()
                 .downsampling(
                     new Downsampling(
-                        List.of(new Round(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("1s"))))
+                        List.of(new Round(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m"))))
                     )
                 )
                 .dataRetention(TimeValue.MAX_VALUE)
@@ -977,7 +977,7 @@ public void testDownsampling() throws Exception {
         String downsampleIndexName = DownsampleConfig.generateDownsampleIndexName(
             DOWNSAMPLED_INDEX_PREFIX,
             state.metadata().index(firstGenIndex),
-            new DateHistogramInterval("1s")
+            new DateHistogramInterval("5m")
         );
         {
             // let's simulate the in-progress downsampling
@@ -1100,7 +1100,7 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc
             DataStreamLifecycle.newBuilder()
                 .downsampling(
                     new Downsampling(
-                        List.of(new Round(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("1s"))))
+                        List.of(new Round(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m"))))
                     )
                 )
                 .dataRetention(TimeValue.MAX_VALUE)
@@ -1129,7 +1129,7 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc
         String downsampleIndexName = DownsampleConfig.generateDownsampleIndexName(
             DOWNSAMPLED_INDEX_PREFIX,
             state.metadata().index(firstGenIndexName),
-            new DateHistogramInterval("1s")
+            new DateHistogramInterval("5m")
         );
         Metadata.Builder newMetadata = Metadata.builder(state.metadata())
             .put(
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java
index b768b468204fc..65de7a7cab265 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java
@@ -322,6 +322,8 @@ public void writeTo(StreamOutput out) throws IOException {
      */
     public record Downsampling(@Nullable List<Round> rounds) implements Writeable, ToXContentFragment {
 
+        public static final long FIVE_MINUTES_MILLIS = TimeValue.timeValueMinutes(5).getMillis();
+
         /**
          * A round represents the configuration for when and how elasticsearch will downsample a backing index.
          * @param after is a TimeValue configuring how old (based on generation age) should a backing index be before downsampling
@@ -356,6 +358,14 @@ public static Round read(StreamInput in) throws IOException {
                 return new Round(in.readTimeValue(), new DownsampleConfig(in));
             }
 
+            public Round {
+                if (config.getFixedInterval().estimateMillis() < FIVE_MINUTES_MILLIS) {
+                    throw new IllegalArgumentException(
+                        "A downsampling round must have a fixed interval of at least five minutes but found: " + config.getFixedInterval()
+                    );
+                }
+            }
+
             @Override
             public void writeTo(StreamOutput out) throws IOException {
                 out.writeTimeValue(after);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java
index e45e0efda636f..441e8491b4b92 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java
@@ -233,6 +233,24 @@ public void testInvalidDownsamplingConfiguration() {
             );
             assertThat(exception.getMessage(), equalTo("Downsampling configuration supports maximum 10 configured rounds. Found: 12"));
         }
+
+        {
+            IllegalArgumentException exception = expectThrows(
+                IllegalArgumentException.class,
+                () -> new DataStreamLifecycle.Downsampling(
+                    List.of(
+                        new DataStreamLifecycle.Downsampling.Round(
+                            TimeValue.timeValueDays(10),
+                            new DownsampleConfig(new DateHistogramInterval("2m"))
+                        )
+                    )
+                )
+            );
+            assertThat(
+                exception.getMessage(),
+                equalTo("A downsampling round must have a fixed interval of at least five minutes but found: 2m")
+            );
+        }
     }
 
     @Nullable
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
index d10e44e3dd1fa..1b1e512113712 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
@@ -1299,15 +1299,15 @@ public void testGetDownsampleRounds() {
                             List.of(
                                 new DataStreamLifecycle.Downsampling.Round(
                                     TimeValue.timeValueMillis(2000),
-                                    new DownsampleConfig(new DateHistogramInterval("10s"))
+                                    new DownsampleConfig(new DateHistogramInterval("10m"))
                                 ),
                                 new DataStreamLifecycle.Downsampling.Round(
                                     TimeValue.timeValueMillis(3200),
-                                    new DownsampleConfig(new DateHistogramInterval("100s"))
+                                    new DownsampleConfig(new DateHistogramInterval("100m"))
                                 ),
                                 new DataStreamLifecycle.Downsampling.Round(
                                     TimeValue.timeValueMillis(3500),
-                                    new DownsampleConfig(new DateHistogramInterval("1000s"))
+                                    new DownsampleConfig(new DateHistogramInterval("1000m"))
                                 )
                             )
                         )
@@ -1360,15 +1360,15 @@ public void testGetDownsampleRounds() {
                             List.of(
                                 new DataStreamLifecycle.Downsampling.Round(
                                     TimeValue.timeValueMillis(2000),
-                                    new DownsampleConfig(new DateHistogramInterval("10s"))
+                                    new DownsampleConfig(new DateHistogramInterval("10m"))
                                 ),
                                 new DataStreamLifecycle.Downsampling.Round(
                                     TimeValue.timeValueMillis(3200),
-                                    new DownsampleConfig(new DateHistogramInterval("100s"))
+                                    new DownsampleConfig(new DateHistogramInterval("100m"))
                                 ),
                                 new DataStreamLifecycle.Downsampling.Round(
                                     TimeValue.timeValueMillis(3500),
-                                    new DownsampleConfig(new DateHistogramInterval("1000s"))
+                                    new DownsampleConfig(new DateHistogramInterval("1000m"))
                                 )
                             )
                         )
diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java
index cef748dfded3c..166f41fa063ca 100644
--- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java
+++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java
@@ -71,7 +71,7 @@ public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception {
                         List.of(
                             new DataStreamLifecycle.Downsampling.Round(
                                 TimeValue.timeValueMillis(0),
-                                new DownsampleConfig(new DateHistogramInterval("1s"))
+                                new DownsampleConfig(new DateHistogramInterval("5m"))
                             )
                         )
                     )
@@ -121,7 +121,7 @@ public boolean validateClusterForming() {
             );
             ensureStableCluster(cluster.numDataAndMasterNodes());
 
-            final String targetIndex = "downsample-1s-" + sourceIndex;
+            final String targetIndex = "downsample-5m-" + sourceIndex;
             assertBusy(() -> {
                 try {
                     GetSettingsResponse getSettingsResponse = client().admin()
diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java
index faa67479cc0d5..cf5e79982d836 100644
--- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java
+++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java
@@ -61,8 +61,8 @@ public void testDownsampling() throws Exception {
             .downsampling(
                 new Downsampling(
                     List.of(
-                        new Downsampling.Round(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("1s"))),
-                        new Downsampling.Round(TimeValue.timeValueSeconds(10), new DownsampleConfig(new DateHistogramInterval("10s")))
+                        new Downsampling.Round(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m"))),
+                        new Downsampling.Round(TimeValue.timeValueSeconds(10), new DownsampleConfig(new DateHistogramInterval("10m")))
                     )
                 )
             )
@@ -72,8 +72,8 @@ public void testDownsampling() throws Exception {
 
         List<String> backingIndices = getBackingIndices(client(), dataStreamName);
         String firstGenerationBackingIndex = backingIndices.get(0);
-        String oneSecondDownsampleIndex = "downsample-1s-" + firstGenerationBackingIndex;
-        String tenSecondsDownsampleIndex = "downsample-10s-" + firstGenerationBackingIndex;
+        String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex;
+        String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex;
 
         Set<String> witnessedDownsamplingIndices = new HashSet<>();
         clusterService().addListener(event -> {
@@ -119,10 +119,10 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception
             .downsampling(
                 new Downsampling(
                     List.of(
-                        new Downsampling.Round(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("1s"))),
+                        new Downsampling.Round(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m"))),
                         // data stream lifecycle runs every 1 second, so by the time we forcemerge the backing index it would've been at
                         // least 2 seconds since rollover. only the 10 seconds round should be executed.
-                        new Downsampling.Round(TimeValue.timeValueMillis(10), new DownsampleConfig(new DateHistogramInterval("10s")))
+                        new Downsampling.Round(TimeValue.timeValueMillis(10), new DownsampleConfig(new DateHistogramInterval("10m")))
                     )
                 )
             )
@@ -131,8 +131,8 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception
 
         List<String> backingIndices = getBackingIndices(client(), dataStreamName);
         String firstGenerationBackingIndex = backingIndices.get(0);
-        String oneSecondDownsampleIndex = "downsample-1s-" + firstGenerationBackingIndex;
-        String tenSecondsDownsampleIndex = "downsample-10s-" + firstGenerationBackingIndex;
+        String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex;
+        String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex;
 
         Set<String> witnessedDownsamplingIndices = new HashSet<>();
         clusterService().addListener(event -> {
@@ -173,10 +173,10 @@ public void testUpdateDownsampleRound() throws Exception {
             .downsampling(
                 new Downsampling(
                     List.of(
-                        new Downsampling.Round(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("1s"))),
+                        new Downsampling.Round(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m"))),
                         // data stream lifecycle runs every 1 second, so by the time we forcemerge the backing index it would've been at
                         // least 2 seconds since rollover. only the 10 seconds round should be executed.
-                        new Downsampling.Round(TimeValue.timeValueMillis(10), new DownsampleConfig(new DateHistogramInterval("10s")))
+                        new Downsampling.Round(TimeValue.timeValueMillis(10), new DownsampleConfig(new DateHistogramInterval("10m")))
                     )
                 )
             )
@@ -186,8 +186,8 @@ public void testUpdateDownsampleRound() throws Exception {
 
         List<String> backingIndices = getBackingIndices(client(), dataStreamName);
         String firstGenerationBackingIndex = backingIndices.get(0);
-        String oneSecondDownsampleIndex = "downsample-1s-" + firstGenerationBackingIndex;
-        String tenSecondsDownsampleIndex = "downsample-10s-" + firstGenerationBackingIndex;
+        String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex;
+        String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex;
 
         Set<String> witnessedDownsamplingIndices = new HashSet<>();
         clusterService().addListener(event -> {
@@ -222,7 +222,7 @@ public void testUpdateDownsampleRound() throws Exception {
         DataStreamLifecycle updatedLifecycle = DataStreamLifecycle.newBuilder()
             .downsampling(
                 new Downsampling(
-                    List.of(new Downsampling.Round(TimeValue.timeValueMillis(10), new DownsampleConfig(new DateHistogramInterval("30s"))))
+                    List.of(new Downsampling.Round(TimeValue.timeValueMillis(10), new DownsampleConfig(new DateHistogramInterval("20m"))))
                 )
             )
             .build();
@@ -232,7 +232,7 @@ public void testUpdateDownsampleRound() throws Exception {
             new PutDataStreamLifecycleAction.Request(new String[] { dataStreamName }, updatedLifecycle)
         );
 
-        String thirtySecondsDownsampleIndex = "downsample-30s-" + firstGenerationBackingIndex;
+        String thirtySecondsDownsampleIndex = "downsample-20m-" + firstGenerationBackingIndex;
 
         assertBusy(() -> {
             assertThat(indexExists(tenSecondsDownsampleIndex), is(false));
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java
index cb3a032f44127..49ac36b854298 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java
@@ -120,11 +120,11 @@ public void testDownsamplingAuthorized() throws Exception {
                     List.of(
                         new DataStreamLifecycle.Downsampling.Round(
                             TimeValue.timeValueMillis(0),
-                            new DownsampleConfig(new DateHistogramInterval("1s"))
+                            new DownsampleConfig(new DateHistogramInterval("5m"))
                         ),
                         new DataStreamLifecycle.Downsampling.Round(
                             TimeValue.timeValueSeconds(10),
-                            new DownsampleConfig(new DateHistogramInterval("10s"))
+                            new DownsampleConfig(new DateHistogramInterval("10m"))
                         )
                     )
                 )
@@ -144,11 +144,11 @@ public void testConfiguringLifecycleWithDownsamplingForSystemDataStreamFails() {
                     List.of(
                         new DataStreamLifecycle.Downsampling.Round(
                             TimeValue.timeValueMillis(0),
-                            new DownsampleConfig(new DateHistogramInterval("1s"))
+                            new DownsampleConfig(new DateHistogramInterval("5m"))
                         ),
                         new DataStreamLifecycle.Downsampling.Round(
                             TimeValue.timeValueSeconds(10),
-                            new DownsampleConfig(new DateHistogramInterval("10s"))
+                            new DownsampleConfig(new DateHistogramInterval("10m"))
                         )
                     )
                 )
@@ -188,8 +188,8 @@ public void testExplicitSystemDataStreamConfigurationWithDownsamplingFails() {
     private void waitAndAssertDownsamplingCompleted(String dataStreamName) throws Exception {
         List<Index> backingIndices = getDataStreamBackingIndices(dataStreamName);
         String firstGenerationBackingIndex = backingIndices.get(0).getName();
-        String oneSecondDownsampleIndex = "downsample-1s-" + firstGenerationBackingIndex;
-        String tenSecondsDownsampleIndex = "downsample-10s-" + firstGenerationBackingIndex;
+        String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex;
+        String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex;
 
         Set<String> witnessedDownsamplingIndices = new HashSet<>();
         clusterService().addListener(event -> {
@@ -439,11 +439,11 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
                         List.of(
                             new DataStreamLifecycle.Downsampling.Round(
                                 TimeValue.timeValueMillis(0),
-                                new DownsampleConfig(new DateHistogramInterval("1s"))
+                                new DownsampleConfig(new DateHistogramInterval("5m"))
                             ),
                             new DataStreamLifecycle.Downsampling.Round(
                                 TimeValue.timeValueSeconds(10),
-                                new DownsampleConfig(new DateHistogramInterval("10s"))
+                                new DownsampleConfig(new DateHistogramInterval("10m"))
                             )
                         )
                     )

From 29f0a233ff5d6b3d139d8d5c672e574e32896a71 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Fri, 15 Sep 2023 21:50:46 +0200
Subject: [PATCH 10/41] Move test to the correct test case. (#99586)

`oe.common.lucene.uid.VersionsTests` is actually about testing the `_version`
field, but the test that is being moved tests `IndexVersion`.
---
 .../common/lucene/uid/VersionsTests.java      | 23 -------------------
 .../index/IndexVersionTests.java              | 22 ++++++++++++++++++
 2 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
index 976add854c584..d6c5fe812140f 100644
--- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.index.IndexVersionUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -198,28 +197,6 @@ public void testCacheFilterReader() throws Exception {
         dir.close();
     }
 
-    public void testLuceneVersionOnUnknownVersions() {
-        // between two known versions, should use the lucene version of the previous version
-        IndexVersion version = IndexVersionUtils.getPreviousVersion();
-        final IndexVersion nextVersion = IndexVersion.fromId(version.id() + 100);
-        if (IndexVersionUtils.allReleasedVersions().contains(nextVersion) == false) {
-            // the version is not known, we make an assumption the Lucene version stays the same
-            assertThat(version.luceneVersion(), equalTo(nextVersion.luceneVersion()));
-        } else {
-            // the version is known, the most we can assert is that the Lucene version is not earlier
-            // Version does not implement Comparable :(
-            assertTrue(nextVersion.luceneVersion().onOrAfter(version.luceneVersion()));
-        }
-
-        // too old version, major should be the oldest supported lucene version minus 1
-        version = IndexVersion.fromId(5020199);
-        assertThat(version.luceneVersion().major, equalTo(IndexVersionUtils.getFirstVersion().luceneVersion().major - 1));
-
-        // future version, should be the same version as today
-        version = IndexVersion.fromId(IndexVersion.current().id() + 100);
-        assertThat(version.luceneVersion(), equalTo(IndexVersion.current().luceneVersion()));
-    }
-
     public void testTimeSeriesLoadDocIdAndVersion() throws Exception {
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
diff --git a/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java b/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java
index 4ffd98eb4d8a9..452da5279f4c1 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexVersionTests.java
@@ -196,4 +196,26 @@ public void testParseLenient() {
             assertThat(luceneVersion, Matchers.equalTo(Lucene.parseVersionLenient(string, null)));
         }
     }
+
+    public void testLuceneVersionOnUnknownVersions() {
+        // between two known versions, should use the lucene version of the previous version
+        IndexVersion version = IndexVersionUtils.getPreviousVersion();
+        final IndexVersion nextVersion = IndexVersion.fromId(version.id() + 100);
+        if (IndexVersionUtils.allReleasedVersions().contains(nextVersion) == false) {
+            // the version is not known, we make an assumption the Lucene version stays the same
+            assertThat(version.luceneVersion(), equalTo(nextVersion.luceneVersion()));
+        } else {
+            // the version is known, the most we can assert is that the Lucene version is not earlier
+            // Version does not implement Comparable :(
+            assertTrue(nextVersion.luceneVersion().onOrAfter(version.luceneVersion()));
+        }
+
+        // too old version, major should be the oldest supported lucene version minus 1
+        version = IndexVersion.fromId(5020199);
+        assertThat(version.luceneVersion().major, equalTo(IndexVersionUtils.getFirstVersion().luceneVersion().major - 1));
+
+        // future version, should be the same version as today
+        version = IndexVersion.fromId(IndexVersion.current().id() + 100);
+        assertThat(version.luceneVersion(), equalTo(IndexVersion.current().luceneVersion()));
+    }
 }

From 397330e099f364263552ca2fa9d75ae2a663ac4c Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Mon, 18 Sep 2023 07:51:44 +0100
Subject: [PATCH 11/41] Adjust other links to unstable cluster docs (#99363)

Relates #99354, #99287.
---
 .../org/elasticsearch/common/reference-docs-links.json        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json
index ed1041acf3f54..4de327d203d16 100644
--- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json
+++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json
@@ -2,8 +2,8 @@
   "INITIAL_MASTER_NODES": "important-settings.html#initial_master_nodes",
   "DISCOVERY_TROUBLESHOOTING": "discovery-troubleshooting.html",
   "UNSTABLE_CLUSTER_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html",
-  "LAGGING_NODE_TROUBLESHOOTING": "cluster-fault-detection.html#_diagnosing_lagging_nodes",
-  "SHARD_LOCK_TROUBLESHOOTING": "cluster-fault-detection.html#_diagnosing_shardlockobtainfailedexception_failures",
+  "LAGGING_NODE_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html#_diagnosing_lagging_nodes_2",
+  "SHARD_LOCK_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html#_diagnosing_shardlockobtainfailedexception_failures_2",
   "CONCURRENT_REPOSITORY_WRITERS": "add-repository.html",
   "ARCHIVE_INDICES": "archive-indices.html",
   "HTTP_TRACER": "modules-network.html#http-rest-request-tracer"

From 5bd64e46fd59b3eb14ad35be47fa45b83680b135 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Mon, 18 Sep 2023 08:27:01 +0100
Subject: [PATCH 12/41] Remove exception-mangling from RecoverySourceHandler
 (#99607)

`RecoverySourceHandler` uses `Listenable*Future` in several places that
do not need the exception-mangling behaviour, so this commit replaces
them with `SubscribableListener`.
---
 .../recovery/RecoverySourceHandler.java       | 70 +++++++++++--------
 1 file changed, 39 insertions(+), 31 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index d00a7e32c1ae1..a3170815251a6 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -19,7 +19,6 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ListenableActionFuture;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
@@ -33,7 +32,6 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.common.util.concurrent.CountDown;
-import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.IOUtils;
@@ -84,6 +82,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.IntSupplier;
@@ -120,7 +119,7 @@ public class RecoverySourceHandler {
     private final RecoveryPlannerService recoveryPlannerService;
     private final CancellableThreads cancellableThreads = new CancellableThreads();
     private final List<Closeable> resources = new CopyOnWriteArrayList<>();
-    private final ListenableActionFuture<RecoveryResponse> future = new ListenableActionFuture<>();
+    private final SubscribableListener<RecoveryResponse> future = new SubscribableListener<>();
 
     public RecoverySourceHandler(
         IndexShard shard,
@@ -231,10 +230,13 @@ && isTargetSameHistory()
             logger.trace("history is retained by retention lock");
         }
 
-        final ListenableFuture<SendFileResult> sendFileStep = new ListenableFuture<>();
-        final ListenableFuture<TimeValue> prepareEngineStep = new ListenableFuture<>();
-        final ListenableFuture<SendSnapshotResult> sendSnapshotStep = new ListenableFuture<>();
-        final ListenableFuture<Void> finalizeStep = new ListenableFuture<>();
+        final SubscribableListener<SendFileResult> sendFileStep = new SubscribableListener<>();
+        final SubscribableListener<TimeValue> prepareEngineStep = new SubscribableListener<>();
+        final SubscribableListener<SendSnapshotResult> sendSnapshotStep = new SubscribableListener<>();
+        final SubscribableListener<Void> finalizeStep = new SubscribableListener<>();
+        final AtomicReference<SendSnapshotResult> sendSnapshotStepResult = new AtomicReference<>();
+        final AtomicReference<SendFileResult> sendFileStepResult = new AtomicReference<>();
+        final AtomicLong prepareEngineTimeMillisRef = new AtomicLong();
 
         if (isSequenceNumberBasedRecovery) {
             logger.trace("performing sequence numbers based recovery. starting at [{}]", request.startingSeqNo());
@@ -294,12 +296,14 @@ && isTargetSameHistory()
 
         sendFileStep.addListener(ActionListener.wrap(r -> {
             assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[prepareTargetForTranslog]");
+            sendFileStepResult.set(r);
             // For a sequence based recovery, the target can keep its local translog
             prepareTargetForTranslog(estimateNumberOfHistoryOperations(startingSeqNo), prepareEngineStep);
         }, onFailure));
 
         prepareEngineStep.addListener(ActionListener.wrap(prepareEngineTime -> {
             assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[phase2]");
+            prepareEngineTimeMillisRef.set(prepareEngineTime.millis());
             /*
              * add shard to replication group (shard will receive replication requests from this point on) now that engine is open.
              * This means that any document indexed into the primary after this will be replicated to this replica as well
@@ -346,14 +350,15 @@ && isTargetSameHistory()
 
         // Recovery target can trim all operations >= startingSeqNo as we have sent all these operations in the phase 2
         final long trimAboveSeqNo = startingSeqNo - 1;
-        sendSnapshotStep.addListener(
-            ActionListener.wrap(r -> finalizeRecovery(r.targetLocalCheckpoint, trimAboveSeqNo, finalizeStep), onFailure)
-        );
+        sendSnapshotStep.addListener(ActionListener.wrap(r -> {
+            sendSnapshotStepResult.set(r);
+            finalizeRecovery(r.targetLocalCheckpoint, trimAboveSeqNo, finalizeStep);
+        }, onFailure));
 
         finalizeStep.addListener(ActionListener.wrap(r -> {
             final long phase1ThrottlingWaitTime = 0L; // TODO: return the actual throttle time
-            final SendSnapshotResult sendSnapshotResult = sendSnapshotStep.result();
-            final SendFileResult sendFileResult = sendFileStep.result();
+            final SendSnapshotResult sendSnapshotResult = sendSnapshotStepResult.get();
+            final SendFileResult sendFileResult = sendFileStepResult.get();
             final RecoveryResponse response = new RecoveryResponse(
                 sendFileResult.phase1FileNames,
                 sendFileResult.phase1FileSizes,
@@ -363,7 +368,7 @@ && isTargetSameHistory()
                 sendFileResult.existingTotalSize,
                 sendFileResult.took.millis(),
                 phase1ThrottlingWaitTime,
-                prepareEngineStep.result().millis(),
+                prepareEngineTimeMillisRef.get(),
                 sendSnapshotResult.sentOperations,
                 sendSnapshotResult.tookTime.millis()
             );
@@ -553,7 +558,7 @@ void phase1(IndexCommit snapshot, long startingSeqNo, IntSupplier translogOps, A
                 logger.trace("skipping [phase1] since source and target have identical sync id [{}]", recoverySourceMetadata.getSyncId());
 
                 // but we must still create a retention lease
-                final ListenableFuture<RetentionLease> createRetentionLeaseStep = new ListenableFuture<>();
+                final SubscribableListener<RetentionLease> createRetentionLeaseStep = new SubscribableListener<>();
                 createRetentionLease(startingSeqNo, createRetentionLeaseStep);
                 createRetentionLeaseStep.addListener(listener.delegateFailureAndWrap((l, retentionLease) -> {
                     final TimeValue took = stopWatch.totalTime();
@@ -661,11 +666,12 @@ void recoverFilesFromSourceAndSnapshot(
         // since the plan can change after a failure recovering files from the snapshots that cannot be
         // recovered from the source node, in that case we have to start from scratch using the fallback
         // recovery plan that would be used in subsequent steps.
-        final ListenableFuture<Void> sendFileInfoStep = new ListenableFuture<>();
-        final ListenableFuture<Tuple<ShardRecoveryPlan, List<StoreFileMetadata>>> recoverSnapshotFilesStep = new ListenableFuture<>();
-        final ListenableFuture<ShardRecoveryPlan> sendFilesStep = new ListenableFuture<>();
-        final ListenableFuture<Tuple<ShardRecoveryPlan, RetentionLease>> createRetentionLeaseStep = new ListenableFuture<>();
-        final ListenableFuture<ShardRecoveryPlan> cleanFilesStep = new ListenableFuture<>();
+        final SubscribableListener<Void> sendFileInfoStep = new SubscribableListener<>();
+        final SubscribableListener<Tuple<ShardRecoveryPlan, List<StoreFileMetadata>>> recoverSnapshotFilesStep =
+            new SubscribableListener<>();
+        final SubscribableListener<ShardRecoveryPlan> sendFilesStep = new SubscribableListener<>();
+        final SubscribableListener<Tuple<ShardRecoveryPlan, RetentionLease>> createRetentionLeaseStep = new SubscribableListener<>();
+        final SubscribableListener<ShardRecoveryPlan> cleanFilesStep = new SubscribableListener<>();
 
         final int translogOps = shardRecoveryPlan.getTranslogOps();
         recoveryTarget.receiveFileInfo(
@@ -790,7 +796,9 @@ private class SnapshotRecoverFileRequestsSender {
         private final CountDown countDown;
         private final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> pendingSnapshotFilesToRecover;
         private final AtomicBoolean cancelled = new AtomicBoolean();
-        private final Set<ListenableFuture<Void>> outstandingRequests = Sets.newHashSetWithExpectedSize(maxConcurrentSnapshotFileDownloads);
+        private final Set<SubscribableListener<Void>> outstandingRequests = Sets.newHashSetWithExpectedSize(
+            maxConcurrentSnapshotFileDownloads
+        );
         private List<StoreFileMetadata> filesFailedToDownloadFromSnapshot;
 
         SnapshotRecoverFileRequestsSender(ShardRecoveryPlan shardRecoveryPlan, ActionListener<List<StoreFileMetadata>> listener) {
@@ -813,7 +821,7 @@ void sendRequest() {
                 return;
             }
 
-            final ListenableFuture<Void> requestFuture = new ListenableFuture<>();
+            final SubscribableListener<Void> requestFuture = new SubscribableListener<>();
             try {
                 cancellableThreads.checkForCancel();
 
@@ -899,7 +907,7 @@ synchronized List<StoreFileMetadata> getFilesFailedToRecoverFromSnapshot() {
             return Objects.requireNonNullElse(filesFailedToDownloadFromSnapshot, Collections.emptyList());
         }
 
-        private void trackOutstandingRequest(ListenableFuture<Void> future) {
+        private void trackOutstandingRequest(SubscribableListener<Void> future) {
             boolean cancelled;
             synchronized (outstandingRequests) {
                 cancelled = cancellableThreads.isCancelled() || this.cancelled.get();
@@ -919,7 +927,7 @@ private void trackOutstandingRequest(ListenableFuture<Void> future) {
             }
         }
 
-        private void unTrackOutstandingRequest(ListenableFuture<Void> future) {
+        private void unTrackOutstandingRequest(SubscribableListener<Void> future) {
             synchronized (outstandingRequests) {
                 outstandingRequests.remove(future);
             }
@@ -928,7 +936,7 @@ private void unTrackOutstandingRequest(ListenableFuture<Void> future) {
         private void notifyFailureOnceAllOutstandingRequestAreDone(Exception e) {
             assert cancelled.get();
 
-            final Set<ListenableFuture<Void>> pendingRequests;
+            final Set<SubscribableListener<Void>> pendingRequests;
             synchronized (outstandingRequests) {
                 pendingRequests = new HashSet<>(outstandingRequests);
             }
@@ -941,7 +949,7 @@ private void notifyFailureOnceAllOutstandingRequestAreDone(Exception e) {
             // new requests and therefore we can safely use to wait until all the pending requests complete
             // to notify the listener about the cancellation
             final CountDown pendingRequestsCountDown = new CountDown(pendingRequests.size());
-            for (ListenableFuture<Void> outstandingFuture : pendingRequests) {
+            for (SubscribableListener<Void> outstandingFuture : pendingRequests) {
                 outstandingFuture.addListener(ActionListener.running(() -> {
                     if (pendingRequestsCountDown.countDown()) {
                         listener.onFailure(e);
@@ -1111,7 +1119,7 @@ void phase2(
         }
         logger.trace("recovery [phase2]: sending transaction log operations (from [" + startingSeqNo + "] to [" + endingSeqNo + "]");
         final StopWatch stopWatch = new StopWatch().start();
-        final ListenableFuture<Void> sendListener = new ListenableFuture<>();
+        final SubscribableListener<Void> sendListener = new SubscribableListener<>();
         final OperationBatchSender sender = new OperationBatchSender(
             startingSeqNo,
             endingSeqNo,
@@ -1252,7 +1260,7 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis
          * marking the shard as in-sync. If the relocation handoff holds all the permits then after the handoff completes and we acquire
          * the permit then the state of the shard will be relocated and this recovery will fail.
          */
-        final ListenableFuture<Void> markInSyncStep = new ListenableFuture<>();
+        final SubscribableListener<Void> markInSyncStep = new SubscribableListener<>();
         runUnderPrimaryPermit(
             () -> shard.markAllocationIdAsInSync(request.targetAllocationId(), targetLocalCheckpoint),
             shard,
@@ -1260,14 +1268,14 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis
             markInSyncStep
         );
 
-        final ListenableFuture<Long> finalizeListener = new ListenableFuture<>();
+        final SubscribableListener<Long> finalizeListener = new SubscribableListener<>();
         markInSyncStep.addListener(listener.delegateFailureAndWrap((l, ignored) -> {
             final long globalCheckpoint = shard.getLastKnownGlobalCheckpoint(); // this global checkpoint is persisted in finalizeRecovery
             cancellableThreads.checkForCancel();
             recoveryTarget.finalizeRecovery(globalCheckpoint, trimAboveSeqNo, finalizeListener.map(ignored2 -> globalCheckpoint));
         }));
 
-        final ListenableFuture<Void> updateGlobalCheckpointStep = new ListenableFuture<>();
+        final SubscribableListener<Void> updateGlobalCheckpointStep = new SubscribableListener<>();
         finalizeListener.addListener(
             listener.delegateFailureAndWrap(
                 (l, globalCheckpoint) -> runUnderPrimaryPermit(
@@ -1279,9 +1287,9 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis
             )
         );
 
-        final ListenableFuture<Void> finalStep;
+        final SubscribableListener<Void> finalStep;
         if (request.isPrimaryRelocation()) {
-            finalStep = new ListenableFuture<>();
+            finalStep = new SubscribableListener<>();
             updateGlobalCheckpointStep.addListener(listener.delegateFailureAndWrap((l, ignored) -> {
                 logger.trace("performing relocation hand-off");
                 cancellableThreads.execute(

From cafa545786423446f0ff070312d51ef769afda2d Mon Sep 17 00:00:00 2001
From: Simon Cooper <simon.cooper@elastic.co>
Date: Mon, 18 Sep 2023 08:46:19 +0100
Subject: [PATCH 13/41] Read index version from the old deployed cluster
 instead of inferring it in full restart tests (#99524)

The index version is needed to check the output against an upgraded cluster. Now the junit tests share a JVM between the old & upgraded clusters, we can store the old index version in the class to be read when needed
---
 ...rameterizedFullClusterRestartTestCase.java | 50 ++++++++++++++++---
 1 file changed, 42 insertions(+), 8 deletions(-)

diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java
index 4609b27db2909..eef8f62eedd98 100644
--- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java
@@ -12,24 +12,30 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
 
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.util.Version;
 import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.ObjectPath;
 import org.junit.AfterClass;
 import org.junit.Before;
 
 import java.util.Arrays;
 import java.util.Locale;
+import java.util.Map;
 
 import static org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus.OLD;
 import static org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus.UPGRADED;
-import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
 
 @TestCaseOrdering(FullClusterRestartTestOrdering.class)
 public abstract class ParameterizedFullClusterRestartTestCase extends ESRestTestCase {
     private static final Version MINIMUM_WIRE_COMPATIBLE_VERSION = Version.fromString("7.17.0");
     private static final Version OLD_CLUSTER_VERSION = Version.fromString(System.getProperty("tests.old_cluster_version"));
+    private static IndexVersion oldIndexVersion;
     private static boolean upgradeFailed = false;
     private static boolean upgraded = false;
     private final FullClusterRestartUpgradeStatus requestedUpgradeStatus;
@@ -43,6 +49,39 @@ public static Iterable<Object[]> parameters() throws Exception {
         return Arrays.stream(FullClusterRestartUpgradeStatus.values()).map(v -> new Object[] { v }).toList();
     }
 
+    @Before
+    public void extractOldIndexVersion() throws Exception {
+        if (upgraded == false) {
+            IndexVersion indexVersion = null;   // these should all be the same version
+
+            Request request = new Request("GET", "_nodes");
+            request.addParameter("filter_path", "nodes.*.index_version,nodes.*.name");
+            Response response = client().performRequest(request);
+            ObjectPath objectPath = ObjectPath.createFromResponse(response);
+            Map<String, Object> nodeMap = objectPath.evaluate("nodes");
+            for (String id : nodeMap.keySet()) {
+                Number ix = objectPath.evaluate("nodes." + id + ".index_version");
+                IndexVersion version;
+                if (ix != null) {
+                    version = IndexVersion.fromId(ix.intValue());
+                } else {
+                    // it doesn't have index version (pre 8.11) - just infer it from the release version
+                    version = IndexVersion.fromId(getOldClusterVersion().id);
+                }
+
+                if (indexVersion == null) {
+                    indexVersion = version;
+                } else {
+                    String name = objectPath.evaluate("nodes." + id + ".name");
+                    assertThat("Node " + name + " has a different index version to other nodes", version, equalTo(indexVersion));
+                }
+            }
+
+            assertThat("Index version could not be read", indexVersion, notNullValue());
+            oldIndexVersion = indexVersion;
+        }
+    }
+
     @Before
     public void maybeUpgrade() throws Exception {
         if (upgraded == false && requestedUpgradeStatus == UPGRADED) {
@@ -81,13 +120,8 @@ public static org.elasticsearch.Version getOldClusterVersion() {
     }
 
     public static IndexVersion getOldClusterIndexVersion() {
-        var version = getOldClusterVersion();
-        if (version.equals(org.elasticsearch.Version.CURRENT)) {
-            return IndexVersion.current();
-        } else {
-            assertThat("Index version needs to be added to restart test parameters", version, lessThan(org.elasticsearch.Version.V_8_11_0));
-            return IndexVersion.fromId(version.id);
-        }
+        assert oldIndexVersion != null;
+        return oldIndexVersion;
     }
 
     public static Version getOldClusterTestVersion() {

From 2d786c05f6e2b4ce928f58c6e2776c37864870bc Mon Sep 17 00:00:00 2001
From: Alexander Spies <alexander.spies@elastic.co>
Date: Mon, 18 Sep 2023 09:46:43 +0200
Subject: [PATCH 14/41] ESQL: Improve log messages (#99470)

- Demote another compute service log message to DEBUG to avoid clutter in production logs.
- Format ESQL query logs more consistently with other production log messages.
---
 docs/changelog/99470.yaml                                    | 5 +++++
 .../xpack/esql/action/EsqlResponseListener.java              | 4 ++--
 2 files changed, 7 insertions(+), 2 deletions(-)
 create mode 100644 docs/changelog/99470.yaml

diff --git a/docs/changelog/99470.yaml b/docs/changelog/99470.yaml
new file mode 100644
index 0000000000000..3e784595cc6ac
--- /dev/null
+++ b/docs/changelog/99470.yaml
@@ -0,0 +1,5 @@
+pr: 99470
+summary: "ESQL: Improve log messages"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java
index 0e0e9a4599780..5ccaeec436f70 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java
@@ -143,12 +143,12 @@ public ActionListener<EsqlQueryResponse> wrapWithLogging() {
         return ActionListener.wrap(r -> {
             onResponse(r);
             // At this point, the StopWatch should already have been stopped, so we log a consistent time.
-            LOGGER.info("Successfully executed ESQL query in {}ms:\n{}", stopWatch.stop().getMillis(), esqlQuery);
+            LOGGER.info("Successfully executed ESQL query in [{}]ms: [{}]", stopWatch.stop().getMillis(), esqlQuery);
         }, ex -> {
             // In case of failure, stop the time manually before sending out the response.
             long timeMillis = stopWatch.stop().getMillis();
             onFailure(ex);
-            LOGGER.info("Failed executing ESQL query in {}ms:\n{}", timeMillis, esqlQuery);
+            LOGGER.info("Failed executing ESQL query in [{}]ms: [{}]", timeMillis, esqlQuery);
         });
     }
 }

From 876827711250b2785d25c0562a255f17eadbc2c0 Mon Sep 17 00:00:00 2001
From: Andrei Dan <andrei.dan@elastic.co>
Date: Mon, 18 Sep 2023 10:30:36 +0100
Subject: [PATCH 15/41] DSL Disable yaml tests for <8.11 as the parser was not
 enabled (#99621)

---
 .../test/data_stream/lifecycle/10_explain_lifecycle.yml   | 4 ++--
 .../lifecycle/190_create_data_stream_with_lifecycle.yml   | 4 ++--
 .../test/data_stream/lifecycle/30_not_found.yml           | 4 ++--
 .../test/cluster.component_template/10_basic.yml          | 8 ++++----
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/10_explain_lifecycle.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/10_explain_lifecycle.yml
index 66901f93f522d..51f2980671add 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/10_explain_lifecycle.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/10_explain_lifecycle.yml
@@ -1,8 +1,8 @@
 ---
 "Explain backing index lifecycle":
   - skip:
-      version: " - 8.9.99"
-      reason: "Explain data stream lifecycle API was updated in 8.10"
+      version: " - 8.10.99"
+      reason: "Data stream lifecycle was GA in 8.11"
       features: allowed_warnings
   - do:
       allowed_warnings:
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/190_create_data_stream_with_lifecycle.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/190_create_data_stream_with_lifecycle.yml
index 7238cebaf1d7b..0e4bbd795c18a 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/190_create_data_stream_with_lifecycle.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/190_create_data_stream_with_lifecycle.yml
@@ -1,8 +1,8 @@
 ---
 "Create data stream with lifecycle":
   - skip:
-      version: " - 8.9.99"
-      reason: "data stream lifecycle in index templates was added after 8.10"
+      version: " - 8.10.99"
+      reason: "Data stream lifecycle was GA in 8.11"
       features: allowed_warnings
   - do:
       allowed_warnings:
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml
index 41f698327bf8c..e0646ba27751e 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml
@@ -1,8 +1,8 @@
 setup:
   - skip:
       features: allowed_warnings
-      version: " - 8.9.99"
-      reason: "Data stream lifecycles only supported in 8.10+"
+      version: " - 8.10.99"
+      reason: "Data stream lifecycle was GA in 8.11"
   - do:
       allowed_warnings:
         - "index template [my-lifecycle] has index patterns [my-data-stream-1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-lifecycle] will take precedence during new index creation"
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
index ab2f522eb3631..500207e969146 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
@@ -117,8 +117,8 @@
 ---
 "Add data stream lifecycle":
   - skip:
-      version: " - 8.9.99"
-      reason: "Data stream lifecycle template was updated after 8.9"
+      version: " - 8.10.99"
+      reason: "Data stream lifecycle was GA in 8.11"
 
   - do:
       cluster.put_component_template:
@@ -145,8 +145,8 @@
 ---
 "Get data stream lifecycle with default rollover":
   - skip:
-      version: " - 8.9.99"
-      reason: "Data stream lifecycle template was added after 8.9"
+      version: " - 8.10.99"
+      reason: "Data stream lifecycle was GA in 8.11"
 
   - do:
       cluster.put_component_template:

From 81b1097bdfaa1439d0f115df64cc979b51e19819 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Mon, 18 Sep 2023 10:32:20 +0100
Subject: [PATCH 16/41] Improve async chaining in RecoverySourceHandler
 (#99619)

Introduces `SubscribableListener#andThen` which encapsulates a very
common pattern for chaining a sequence of async actions together. With
this utility you can write the actions in their logical order, without
needing to introduce extra local variables for the back-references.

Also adapts some of the async action chains in `RecoverySourceHandler`
to use this new utility.
---
 .../action/support/SubscribableListener.java  |  48 +++
 .../recovery/RecoverySourceHandler.java       | 286 +++++++++---------
 .../support/SubscribableListenerTests.java    | 112 +++++++
 3 files changed, 302 insertions(+), 144 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java b/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java
index 5ba43111b4f03..cebb4ed6e06e6 100644
--- a/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java
+++ b/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java
@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
@@ -333,6 +334,53 @@ public void complete(ActionListener<?> listener) {
         }
     }
 
+    /**
+     * Creates and returns a new {@link SubscribableListener} {@code L} and subscribes {@code nextStep} to this listener such that if this
+     * listener is completed successfully with result {@code R} then {@code nextStep} is invoked with arguments {@code L} and {@code R}. If
+     * this listener is completed with exception {@code E} then so is {@code L}.
+     * <p>
+     * This can be used to construct a sequence of async actions, each invoked with the result of the previous one:
+     * <pre>
+     * l.andThen((l1, o1) -> forkAction1(o1, args1, l1)).andThen((l2, o2) -> forkAction2(o2, args2, l2)).addListener(finalListener);
+     * </pre>
+     * After creating this chain, completing {@code l} with a successful response will pass the response to {@code forkAction1}, which will
+     * on completion pass its response to {@code forkAction2}, which will in turn pass its response to {@code finalListener}. A failure of
+     * any step will bypass the remaining steps and ultimately fail {@code finalListener}.
+     * <p>
+     * The threading of the {@code nextStep} callback is the same as for listeners added with {@link #addListener}: if this listener is
+     * already complete then {@code nextStep} is invoked on the thread calling {@link #andThen} and in its thread context, but if this
+     * listener is incomplete then {@code nextStep} is invoked on the completing thread and in its thread context.
+     */
+    public <U> SubscribableListener<U> andThen(CheckedBiConsumer<ActionListener<U>, T, ? extends Exception> nextStep) {
+        return andThen(EsExecutors.DIRECT_EXECUTOR_SERVICE, null, nextStep);
+    }
+
+    /**
+     * Creates and returns a new {@link SubscribableListener} {@code L} and subscribes {@code nextStep} to this listener such that if this
+     * listener is completed successfully with result {@code R} then {@code nextStep} is invoked with arguments {@code L} and {@code R}. If
+     * this listener is completed with exception {@code E} then so is {@code L}.
+     * <p>
+     * This can be used to construct a sequence of async actions, each invoked with the result of the previous one:
+     * <pre>
+     * l.andThen(x, t, (l1,o1) -> forkAction1(o1,args1,l1)).andThen(x, t, (l2,o2) -> forkAction2(o2,args2,l2)).addListener(finalListener);
+     * </pre>
+     * After creating this chain, completing {@code l} with a successful response will pass the response to {@code forkAction1}, which will
+     * on completion pass its response to {@code forkAction2}, which will in turn pass its response to {@code finalListener}. A failure of
+     * any step will bypass the remaining steps and ultimately fail {@code finalListener}.
+     * <p>
+     * The threading of the {@code nextStep} callback is the same as for listeners added with {@link #addListener}: if this listener is
+     * already complete then {@code nextStep} is invoked on the thread calling {@link #andThen} and in its thread context, but if this
+     * listener is incomplete then {@code nextStep} is invoked using {@code executor}, in a thread context captured when {@link #andThen}
+     * was called.
+     */
+    public <U> SubscribableListener<U> andThen(
+        Executor executor,
+        @Nullable ThreadContext threadContext,
+        CheckedBiConsumer<ActionListener<U>, T, ? extends Exception> nextStep
+    ) {
+        return newForked(l -> addListener(l.delegateFailureAndWrap(nextStep), executor, threadContext));
+    }
+
     /**
      * Adds a timeout to this listener, such that if the timeout elapses before the listener is completed then it will be completed with an
      * {@link ElasticsearchTimeoutException}.
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index a3170815251a6..1acc0f1041e39 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -39,7 +39,6 @@
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.RecoveryEngineException;
@@ -541,41 +540,49 @@ void phase1(IndexCommit snapshot, long startingSeqNo, IntSupplier translogOps, A
             // TODO: is this still relevant today?
             if (hasSameLegacySyncId(recoverySourceMetadata, request.metadataSnapshot()) == false) {
                 cancellableThreads.checkForCancel();
-                final boolean canUseSnapshots = canUseSnapshots();
-                recoveryPlannerService.computeRecoveryPlan(
-                    shard.shardId(),
-                    shardStateIdentifier,
-                    recoverySourceMetadata,
-                    request.metadataSnapshot(),
-                    startingSeqNo,
-                    translogOps.getAsInt(),
-                    getRequest().targetNode().getMaxIndexVersion(),
-                    canUseSnapshots,
-                    request.isPrimaryRelocation(),
-                    listener.delegateFailureAndWrap((l, plan) -> recoverFilesFromSourceAndSnapshot(plan, store, stopWatch, l))
-                );
+                SubscribableListener
+                    // compute the plan
+                    .<ShardRecoveryPlan>newForked(
+                        l -> recoveryPlannerService.computeRecoveryPlan(
+                            shard.shardId(),
+                            shardStateIdentifier,
+                            recoverySourceMetadata,
+                            request.metadataSnapshot(),
+                            startingSeqNo,
+                            translogOps.getAsInt(),
+                            getRequest().targetNode().getMaxIndexVersion(),
+                            canUseSnapshots(),
+                            request.isPrimaryRelocation(),
+                            l
+                        )
+                    )
+                    // perform the file recovery
+                    .<SendFileResult>andThen((l, plan) -> recoverFilesFromSourceAndSnapshot(plan, store, stopWatch, l))
+                    // and respond
+                    .addListener(listener);
             } else {
                 logger.trace("skipping [phase1] since source and target have identical sync id [{}]", recoverySourceMetadata.getSyncId());
-
-                // but we must still create a retention lease
-                final SubscribableListener<RetentionLease> createRetentionLeaseStep = new SubscribableListener<>();
-                createRetentionLease(startingSeqNo, createRetentionLeaseStep);
-                createRetentionLeaseStep.addListener(listener.delegateFailureAndWrap((l, retentionLease) -> {
-                    final TimeValue took = stopWatch.totalTime();
-                    logger.trace("recovery [phase1]: took [{}]", took);
-                    l.onResponse(
-                        new SendFileResult(
-                            Collections.emptyList(),
-                            Collections.emptyList(),
-                            0L,
-                            Collections.emptyList(),
-                            Collections.emptyList(),
-                            0L,
-                            took
-                        )
-                    );
-                }));
-
+                SubscribableListener
+                    // but we must still create a retention lease
+                    .<RetentionLease>newForked(leaseListener -> createRetentionLease(startingSeqNo, leaseListener))
+                    // and then compute the result of sending no files
+                    .<SendFileResult>andThen((l, ignored) -> {
+                        final TimeValue took = stopWatch.totalTime();
+                        logger.trace("recovery [phase1]: took [{}]", took);
+                        l.onResponse(
+                            new SendFileResult(
+                                Collections.emptyList(),
+                                Collections.emptyList(),
+                                0L,
+                                Collections.emptyList(),
+                                Collections.emptyList(),
+                                0L,
+                                took
+                            )
+                        );
+                    })
+                    // and finally respond
+                    .addListener(listener);
             }
         } catch (Exception e) {
             throw new RecoverFilesRecoveryException(request.shardId(), 0, ByteSizeValue.ZERO, e);
@@ -594,15 +601,6 @@ void recoverFilesFromSourceAndSnapshot(
         StopWatch stopWatch,
         ActionListener<SendFileResult> listener
     ) {
-        cancellableThreads.checkForCancel();
-
-        final List<String> filesToRecoverNames = shardRecoveryPlan.getFilesToRecoverNames();
-        final List<Long> filesToRecoverSizes = shardRecoveryPlan.getFilesToRecoverSizes();
-        final List<String> phase1ExistingFileNames = shardRecoveryPlan.getFilesPresentInTargetNames();
-        final List<Long> phase1ExistingFileSizes = shardRecoveryPlan.getFilesPresentInTargetSizes();
-        final long totalSize = shardRecoveryPlan.getTotalSize();
-        final long existingTotalSize = shardRecoveryPlan.getExistingSize();
-
         if (logger.isTraceEnabled()) {
             for (StoreFileMetadata md : shardRecoveryPlan.getFilesPresentInTarget()) {
                 logger.trace(
@@ -640,6 +638,9 @@ void recoverFilesFromSourceAndSnapshot(
                 }
             }
 
+            final long totalSize = shardRecoveryPlan.getTotalSize();
+            final long existingTotalSize = shardRecoveryPlan.getExistingSize();
+
             logger.trace(
                 """
                     recovery [phase1]: total_size[{}], \
@@ -657,121 +658,118 @@ void recoverFilesFromSourceAndSnapshot(
                         .mapToLong(BlobStoreIndexShardSnapshot.FileInfo::length)
                         .sum()
                 ),
-                phase1ExistingFileNames.size(),
+                shardRecoveryPlan.getFilesPresentInTarget().size(),
                 ByteSizeValue.ofBytes(existingTotalSize)
             );
         }
 
-        // We need to pass the ShardRecovery plan between steps instead of capturing it in the closures
+        new FileBasedRecoveryContext(store, stopWatch, shardRecoveryPlan).run(listener);
+    }
+
+    private class FileBasedRecoveryContext {
+        private final Store store;
+        private final StopWatch stopWatch;
+        private final int translogOps;
+
+        // We need to mutate the ShardRecovery plan instead of capturing it in the closures
         // since the plan can change after a failure recovering files from the snapshots that cannot be
         // recovered from the source node, in that case we have to start from scratch using the fallback
         // recovery plan that would be used in subsequent steps.
-        final SubscribableListener<Void> sendFileInfoStep = new SubscribableListener<>();
-        final SubscribableListener<Tuple<ShardRecoveryPlan, List<StoreFileMetadata>>> recoverSnapshotFilesStep =
-            new SubscribableListener<>();
-        final SubscribableListener<ShardRecoveryPlan> sendFilesStep = new SubscribableListener<>();
-        final SubscribableListener<Tuple<ShardRecoveryPlan, RetentionLease>> createRetentionLeaseStep = new SubscribableListener<>();
-        final SubscribableListener<ShardRecoveryPlan> cleanFilesStep = new SubscribableListener<>();
-
-        final int translogOps = shardRecoveryPlan.getTranslogOps();
-        recoveryTarget.receiveFileInfo(
-            filesToRecoverNames,
-            filesToRecoverSizes,
-            phase1ExistingFileNames,
-            phase1ExistingFileSizes,
-            translogOps,
-            sendFileInfoStep
-        );
-
-        sendFileInfoStep.addListener(
-            listener.delegateFailureAndWrap((l, unused) -> recoverSnapshotFiles(shardRecoveryPlan, new ActionListener<>() {
-                @Override
-                public void onResponse(List<StoreFileMetadata> filesFailedToRecoverFromSnapshot) {
-                    recoverSnapshotFilesStep.onResponse(Tuple.tuple(shardRecoveryPlan, filesFailedToRecoverFromSnapshot));
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    if (shardRecoveryPlan.canRecoverSnapshotFilesFromSourceNode() == false
-                        && e instanceof CancellableThreads.ExecutionCancelledException == false) {
-                        ShardRecoveryPlan fallbackPlan = shardRecoveryPlan.getFallbackPlan();
-                        recoveryTarget.receiveFileInfo(
-                            fallbackPlan.getFilesToRecoverNames(),
-                            fallbackPlan.getFilesToRecoverSizes(),
-                            fallbackPlan.getFilesPresentInTargetNames(),
-                            fallbackPlan.getFilesPresentInTargetSizes(),
-                            fallbackPlan.getTranslogOps(),
-                            recoverSnapshotFilesStep.map(r -> Tuple.tuple(fallbackPlan, Collections.emptyList()))
-                        );
-                    } else {
-                        recoverSnapshotFilesStep.onFailure(e);
-                    }
-                }
-            }))
-        );
+        private ShardRecoveryPlan shardRecoveryPlan;
 
-        recoverSnapshotFilesStep.addListener(listener.delegateFailureAndWrap((l, planAndFilesFailedToRecoverFromSnapshot) -> {
-            ShardRecoveryPlan recoveryPlan = planAndFilesFailedToRecoverFromSnapshot.v1();
-            List<StoreFileMetadata> filesFailedToRecoverFromSnapshot = planAndFilesFailedToRecoverFromSnapshot.v2();
-            final List<StoreFileMetadata> filesToRecoverFromSource;
-            if (filesFailedToRecoverFromSnapshot.isEmpty()) {
-                filesToRecoverFromSource = recoveryPlan.getSourceFilesToRecover();
-            } else {
-                filesToRecoverFromSource = concatLists(recoveryPlan.getSourceFilesToRecover(), filesFailedToRecoverFromSnapshot);
-            }
+        FileBasedRecoveryContext(Store store, StopWatch stopWatch, ShardRecoveryPlan shardRecoveryPlan) {
+            this.store = store;
+            this.stopWatch = stopWatch;
+            this.translogOps = shardRecoveryPlan.getTranslogOps();
+            this.shardRecoveryPlan = shardRecoveryPlan;
+        }
 
-            sendFiles(
-                store,
-                filesToRecoverFromSource.toArray(new StoreFileMetadata[0]),
-                recoveryPlan::getTranslogOps,
-                sendFilesStep.map(unused -> recoveryPlan)
+        private void sendShardRecoveryPlanFileInfo(ActionListener<Void> fileInfoListener) {
+            recoveryTarget.receiveFileInfo(
+                shardRecoveryPlan.getFilesToRecoverNames(),
+                shardRecoveryPlan.getFilesToRecoverSizes(),
+                shardRecoveryPlan.getFilesPresentInTargetNames(),
+                shardRecoveryPlan.getFilesPresentInTargetSizes(),
+                shardRecoveryPlan.getTranslogOps(),
+                fileInfoListener
             );
-        }));
+        }
 
-        sendFilesStep.addListener(
-            listener.delegateFailureAndWrap(
-                (l, recoveryPlan) -> createRetentionLease(
-                    recoveryPlan.getStartingSeqNo(),
-                    createRetentionLeaseStep.map(retentionLease -> Tuple.tuple(recoveryPlan, retentionLease))
-                )
-            )
-        );
+        void run(ActionListener<SendFileResult> listener) {
+            cancellableThreads.checkForCancel();
 
-        createRetentionLeaseStep.addListener(listener.delegateFailureAndWrap((l, recoveryPlanAndRetentionLease) -> {
-            final ShardRecoveryPlan recoveryPlan = recoveryPlanAndRetentionLease.v1();
-            final RetentionLease retentionLease = recoveryPlanAndRetentionLease.v2();
-            final Store.MetadataSnapshot recoverySourceMetadata = recoveryPlan.getSourceMetadataSnapshot();
-            final long lastKnownGlobalCheckpoint = shard.getLastKnownGlobalCheckpoint();
-            assert retentionLease == null || retentionLease.retainingSequenceNumber() - 1 <= lastKnownGlobalCheckpoint
-                : retentionLease + " vs " + lastKnownGlobalCheckpoint;
-            // Establishes new empty translog on the replica with global checkpoint set to lastKnownGlobalCheckpoint. We want
-            // the commit we just copied to be a safe commit on the replica, so why not set the global checkpoint on the replica
-            // to the max seqno of this commit? Because (in rare corner cases) this commit might not be a safe commit here on
-            // the primary, and in these cases the max seqno would be too high to be valid as a global checkpoint.
-            cleanFiles(
-                store,
-                recoverySourceMetadata,
-                () -> translogOps,
-                lastKnownGlobalCheckpoint,
-                cleanFilesStep.map(unused -> recoveryPlan)
-            );
-        }));
+            SubscribableListener
+                // send the original plan
+                .newForked(this::sendShardRecoveryPlanFileInfo)
+                // instruct the target to recover files from snapshot, possibly updating the plan on failure
+                .<List<StoreFileMetadata>>andThen(
+                    (l, ignored) -> recoverSnapshotFiles(shardRecoveryPlan, l.delegateResponse((recoverSnapshotFilesListener, e) -> {
+                        if (shardRecoveryPlan.canRecoverSnapshotFilesFromSourceNode() == false
+                            && e instanceof CancellableThreads.ExecutionCancelledException == false) {
+                            shardRecoveryPlan = shardRecoveryPlan.getFallbackPlan();
+                            sendShardRecoveryPlanFileInfo(recoverSnapshotFilesListener.map(r -> Collections.emptyList()));
+                        } else {
+                            recoverSnapshotFilesListener.onFailure(e);
+                        }
+                    }))
+                )
+                // send local files which either aren't in the snapshot, or which failed to be recovered from the snapshot for some reason
+                .<Void>andThen((sendFilesListener, filesFailedToRecoverFromSnapshot) -> {
+                    final List<StoreFileMetadata> filesToRecoverFromSource;
+                    if (filesFailedToRecoverFromSnapshot.isEmpty()) {
+                        filesToRecoverFromSource = shardRecoveryPlan.getSourceFilesToRecover();
+                    } else {
+                        filesToRecoverFromSource = concatLists(
+                            shardRecoveryPlan.getSourceFilesToRecover(),
+                            filesFailedToRecoverFromSnapshot
+                        );
+                    }
 
-        cleanFilesStep.addListener(listener.delegateFailureAndWrap((l, recoveryPlan) -> {
-            final TimeValue took = stopWatch.totalTime();
-            logger.trace("recovery [phase1]: took [{}]", took);
-            l.onResponse(
-                new SendFileResult(
-                    recoveryPlan.getFilesToRecoverNames(),
-                    recoveryPlan.getFilesToRecoverSizes(),
-                    recoveryPlan.getTotalSize(),
-                    recoveryPlan.getFilesPresentInTargetNames(),
-                    recoveryPlan.getFilesPresentInTargetSizes(),
-                    recoveryPlan.getExistingSize(),
-                    took
+                    sendFiles(
+                        store,
+                        filesToRecoverFromSource.toArray(new StoreFileMetadata[0]),
+                        shardRecoveryPlan::getTranslogOps,
+                        sendFilesListener
+                    );
+                })
+                // create a retention lease
+                .<RetentionLease>andThen(
+                    (createRetentionLeaseListener, ignored) -> createRetentionLease(
+                        shardRecoveryPlan.getStartingSeqNo(),
+                        createRetentionLeaseListener
+                    )
                 )
-            );
-        }));
+                // run cleanFiles, renaming temp files, removing surplus ones, creating an empty translog and so on
+                .<Void>andThen((finalRecoveryPlanListener, retentionLease) -> {
+                    final Store.MetadataSnapshot recoverySourceMetadata = shardRecoveryPlan.getSourceMetadataSnapshot();
+                    final long lastKnownGlobalCheckpoint = shard.getLastKnownGlobalCheckpoint();
+                    assert retentionLease == null || retentionLease.retainingSequenceNumber() - 1 <= lastKnownGlobalCheckpoint
+                        : retentionLease + " vs " + lastKnownGlobalCheckpoint;
+                    // Establishes new empty translog on the replica with global checkpoint set to lastKnownGlobalCheckpoint. We want
+                    // the commit we just copied to be a safe commit on the replica, so why not set the global checkpoint on the replica
+                    // to the max seqno of this commit? Because (in rare corner cases) this commit might not be a safe commit here on
+                    // the primary, and in these cases the max seqno would be too high to be valid as a global checkpoint.
+                    cleanFiles(store, recoverySourceMetadata, () -> translogOps, lastKnownGlobalCheckpoint, finalRecoveryPlanListener);
+                })
+                // compute the result
+                .<SendFileResult>andThen((resultListener, ignored) -> {
+                    final TimeValue took = stopWatch.totalTime();
+                    logger.trace("recovery [phase1]: took [{}]", took);
+                    resultListener.onResponse(
+                        new SendFileResult(
+                            shardRecoveryPlan.getFilesToRecoverNames(),
+                            shardRecoveryPlan.getFilesToRecoverSizes(),
+                            shardRecoveryPlan.getTotalSize(),
+                            shardRecoveryPlan.getFilesPresentInTargetNames(),
+                            shardRecoveryPlan.getFilesPresentInTargetSizes(),
+                            shardRecoveryPlan.getExistingSize(),
+                            took
+                        )
+                    );
+                })
+                // and finally respond
+                .addListener(listener);
+        }
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java
index 9ef5e40e1f932..419cac9a0255d 100644
--- a/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/SubscribableListenerTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -398,4 +399,115 @@ public void testCreateUtils() throws Exception {
         assertTrue(forkFailed.isDone());
         assertEquals("simulated fork failure", expectThrows(ElasticsearchException.class, forkFailed::rawResult).getMessage());
     }
+
+    public void testAndThenSuccess() {
+        final var initialListener = new SubscribableListener<>();
+        final var forked = new AtomicReference<ActionListener<Object>>();
+        final var result = new AtomicReference<>();
+
+        final var chainedListener = initialListener.andThen((l, o) -> {
+            forked.set(l);
+            result.set(o);
+        });
+        assertNull(forked.get());
+        assertNull(result.get());
+
+        final var o1 = new Object();
+        initialListener.onResponse(o1);
+        assertSame(o1, result.get());
+        assertSame(chainedListener, forked.get());
+        assertFalse(chainedListener.isDone());
+    }
+
+    public void testAndThenFailure() {
+        final var initialListener = new SubscribableListener<>();
+
+        final var chainedListener = initialListener.andThen((l, o) -> fail("should not be called"));
+        assertFalse(chainedListener.isDone());
+
+        initialListener.onFailure(new ElasticsearchException("simulated"));
+        assertComplete(chainedListener, "simulated");
+    }
+
+    public void testAndThenThreading() {
+        runAndThenThreadingTest(true);
+        runAndThenThreadingTest(false);
+    }
+
+    private static void runAndThenThreadingTest(boolean testSuccess) {
+        final var completeListener = testSuccess
+            ? SubscribableListener.newSucceeded(new Object())
+            : SubscribableListener.newFailed(new ElasticsearchException("immediate failure"));
+
+        assertComplete(
+            completeListener.andThen(r -> fail("should not fork"), null, ActionListener::onResponse),
+            testSuccess ? null : "immediate failure"
+        );
+
+        final var threadContext = new ThreadContext(Settings.EMPTY);
+        final var headerName = "test-header";
+        final var expectedHeader = randomAlphaOfLength(10);
+
+        final SubscribableListener<Object> deferredListener = new SubscribableListener<>();
+        final SubscribableListener<Object> chainedListener;
+        final AtomicReference<Runnable> forkedRunnable = new AtomicReference<>();
+
+        try (var ignored = threadContext.stashContext()) {
+            threadContext.putHeader(headerName, expectedHeader);
+            chainedListener = deferredListener.andThen(
+                r -> assertTrue(forkedRunnable.compareAndSet(null, r)),
+                threadContext,
+                (l, response) -> {
+                    assertEquals(expectedHeader, threadContext.getHeader(headerName));
+                    l.onResponse(response);
+                }
+            );
+        }
+
+        assertFalse(chainedListener.isDone());
+        assertNull(forkedRunnable.get());
+
+        final AtomicBoolean isComplete = new AtomicBoolean();
+
+        try (var ignored = threadContext.stashContext()) {
+            threadContext.putHeader(headerName, randomAlphaOfLength(10));
+            chainedListener.addListener(ActionListener.running(() -> {
+                assertEquals(expectedHeader, threadContext.getHeader(headerName));
+                assertTrue(isComplete.compareAndSet(false, true));
+            }));
+        }
+
+        try (var ignored = threadContext.stashContext()) {
+            threadContext.putHeader(headerName, randomAlphaOfLength(10));
+            if (testSuccess) {
+                deferredListener.onResponse(new Object());
+            } else {
+                deferredListener.onFailure(new ElasticsearchException("simulated failure"));
+            }
+        }
+
+        assertFalse(chainedListener.isDone());
+        assertFalse(isComplete.get());
+
+        try (var ignored = threadContext.stashContext()) {
+            threadContext.putHeader(headerName, randomAlphaOfLength(10));
+            forkedRunnable.get().run();
+        }
+
+        assertComplete(chainedListener, testSuccess ? null : "simulated failure");
+        assertTrue(isComplete.get());
+    }
+
+    private static void assertComplete(SubscribableListener<Object> listener, @Nullable String expectedFailureMessage) {
+        assertTrue(listener.isDone());
+        if (expectedFailureMessage == null) {
+            try {
+                listener.rawResult();
+            } catch (Exception e) {
+                throw new AssertionError("unexpected", e);
+            }
+        } else {
+            assertEquals(expectedFailureMessage, expectThrows(ElasticsearchException.class, listener::rawResult).getMessage());
+        }
+    }
 }

From 986b53732789a051584a2a0449923d5d7b109d2a Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Mon, 18 Sep 2023 13:29:48 +0300
Subject: [PATCH 17/41] Prepare downsample plugin rest tests for serverless
 (#99622)

Following instructions in [README](https://github.com/elastic/elasticsearch-serverless/blob/main/qa/stateful/README.md)
as well as mimicking changes in #99413:

 * Use the new test-clusters framework using JUnit rules, not the
 "legacy" Gradle plugin.
 * Ensure that the elasticsearch.internal-test-artifact, plugin is
 applied to the project build script.
 * Remove setting the number of replicas in yaml tests.
---
 x-pack/plugin/downsample/qa/rest/build.gradle | 19 ++++++++++++-------
 .../xpack/downsample/DownsampleRestIT.java    | 15 +++++++++++++++
 .../test/downsample/10_basic.yml              | 14 --------------
 .../test/downsample/20_unsupported_aggs.yml   |  1 -
 .../test/downsample/30_date_histogram.yml     |  1 -
 .../test/downsample/40_runtime_fields.yml     |  3 ---
 .../downsample/50_auto_date_histogram.yml     |  2 --
 .../test/downsample/60_settings.yml           |  1 -
 8 files changed, 27 insertions(+), 29 deletions(-)

diff --git a/x-pack/plugin/downsample/qa/rest/build.gradle b/x-pack/plugin/downsample/qa/rest/build.gradle
index 1cffa7fcafc6a..603d69d695cac 100644
--- a/x-pack/plugin/downsample/qa/rest/build.gradle
+++ b/x-pack/plugin/downsample/qa/rest/build.gradle
@@ -5,13 +5,14 @@
  * 2.0.
  */
 
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.legacy-yaml-rest-test'
-apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-test-artifact'
 
 dependencies {
+  testImplementation project(path: ':test:test-clusters')
   yamlRestTestImplementation project(path: xpackModule('rollup'))
 }
 
@@ -21,12 +22,16 @@ restResources {
   }
 }
 
-testClusters.configureEach {
-  testDistribution = 'DEFAULT'
-  setting 'xpack.license.self_generated.type', 'basic'
-  setting 'xpack.security.enabled', 'false'
+artifacts {
+  restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
 }
 
+tasks.named('yamlRestTest') {
+  usesDefaultDistribution()
+}
+tasks.named('yamlRestTestV7CompatTest') {
+  usesDefaultDistribution()
+}
 if (BuildParams.inFipsJvm){
   // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC
   tasks.named("yamlRestTest").configure{enabled = false }
diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java
index 5a66a97d16e6e..504326f1bd4b1 100644
--- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java
@@ -9,11 +9,26 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 public class DownsampleRestIT extends ESClientYamlSuiteTestCase {
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .setting("xpack.license.self_generated.type", "basic")
+        .setting("xpack.security.enabled", "false")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
     public DownsampleRestIT(final ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml
index 70076562b27dd..f4dd187b82445 100644
--- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml
@@ -9,7 +9,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [metricset, k8s.pod.uid]
@@ -92,7 +91,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ metricset, k8s.pod.uid ]
@@ -159,7 +157,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ metricset, k8s.pod.uid ]
@@ -225,7 +222,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ metricset, k8s.pod.uid ]
@@ -343,7 +339,6 @@ setup:
   - match: { test-downsample.settings.index.routing_path: [ "metricset", "k8s.pod.uid"] }
   - match: { test-downsample.settings.index.downsample.source.name: test }
   - match: { test-downsample.settings.index.number_of_shards: "1" }
-  - match: { test-downsample.settings.index.number_of_replicas: "0" }
 
   # Assert rollup index mapping
   - do:
@@ -373,7 +368,6 @@ setup:
       indices.segments:
         index: test-downsample
 
-  - match:   { _shards.total: 1}
   - match:   { indices.test-downsample.shards.0.0.num_committed_segments: 1}
   - match:   { indices.test-downsample.shards.0.0.num_search_segments: 1}
 
@@ -424,7 +418,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ metricset, k8s.pod.uid ]
@@ -438,7 +431,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ metricset, k8s.pod.uid ]
@@ -515,7 +507,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [metricset]
@@ -609,7 +600,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [metricset]
@@ -935,7 +925,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ metricset, k8s.pod.uid ]
@@ -1038,7 +1027,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ metricset, k8s.pod.uid ]
@@ -1141,7 +1129,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ metricset, k8s.pod.uid ]
@@ -1377,7 +1364,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [metricset, k8s.pod.uid]
diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/20_unsupported_aggs.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/20_unsupported_aggs.yml
index f58985189b9e0..bf5f92f628444 100644
--- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/20_unsupported_aggs.yml
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/20_unsupported_aggs.yml
@@ -9,7 +9,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ uid ]
diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml
index b0a6f699a6ac0..b7f3ec7b8f384 100644
--- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml
@@ -9,7 +9,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ uid ]
diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml
index 1d5221fb8d1f1..06d74494e89c7 100644
--- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml
@@ -11,7 +11,6 @@
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ metricset, k8s.pod.uid ]
@@ -173,7 +172,6 @@
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ metricset, k8s.pod.uid ]
@@ -295,7 +293,6 @@
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ metricset, k8s.pod.uid ]
diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/50_auto_date_histogram.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/50_auto_date_histogram.yml
index d314be1f54739..050b16f0674ca 100644
--- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/50_auto_date_histogram.yml
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/50_auto_date_histogram.yml
@@ -9,7 +9,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ uid ]
@@ -33,7 +32,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [ uid ]
diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml
index a9b72f7f73c6a..6a33cc47e5c51 100644
--- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml
@@ -25,7 +25,6 @@
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               default_pipeline: "pipeline"
               final_pipeline: "pipeline"

From 3691312acad38d63d8fe62655d2fd7f56fbeacb8 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Mon, 18 Sep 2023 13:26:03 +0100
Subject: [PATCH 18/41] Slightly adjust docs about S3 incompatibilities
 (#99624)

It's often useful to quote these docs to users encountering problems
with their not-quite-S3-compatible storage system. In practice we don't
need to quote the bits in the middle but we do need the last sentence
about working with the supplier to address incompatibilities. This
commit reorders things so that the most commonly quoted sentences form a
standalone paragraph.
---
 docs/reference/snapshot-restore/repository-s3.asciidoc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc
index c34b295f9bd1a..16171e2e71631 100644
--- a/docs/reference/snapshot-restore/repository-s3.asciidoc
+++ b/docs/reference/snapshot-restore/repository-s3.asciidoc
@@ -226,16 +226,16 @@ emulate S3's behaviour in full. The `repository-s3` type requires full
 compatibility with S3. In particular it must support the same set of API
 endpoints, return the same errors in case of failures, and offer consistency
 and performance at least as good as S3 even when accessed concurrently by
-multiple nodes. Incompatible error codes, consistency or performance may be
-particularly hard to track down since errors, consistency failures, and
-performance issues are usually rare and hard to reproduce.
+multiple nodes. You will need to work with the supplier of your storage system
+to address any incompatibilities you encounter.
 
 You can perform some basic checks of the suitability of your storage system
 using the {ref}/repo-analysis-api.html[repository analysis API]. If this API
 does not complete successfully, or indicates poor performance, then your
 storage system is not fully compatible with AWS S3 and therefore unsuitable for
-use as a snapshot repository. You will need to work with the supplier of your
-storage system to address any incompatibilities you encounter.
+use as a snapshot repository. However, these checks do not guarantee full
+compatibility. Incompatible error codes and consistency or performance issues
+may be rare and hard to reproduce.
 
 [[repository-s3-repository]]
 ==== Repository settings

From 6c26ca0751fada36b71f14d4fc266d6cfcea84f0 Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Mon, 18 Sep 2023 10:03:03 -0500
Subject: [PATCH 19/41] Minor clean up of guava dependencies (#99593)

This commit bumps up the version of jimfs and jimfs_gauva and guava
that is used for security and idp plugins.
---
 build-tools-internal/version.properties      |  4 +--
 gradle/verification-metadata.xml             | 26 ++++++++++++++------
 qa/evil-tests/build.gradle                   |  2 +-
 x-pack/plugin/identity-provider/build.gradle |  3 +--
 x-pack/plugin/security/build.gradle          |  3 +--
 5 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
index ce191115b82f8..274f6a98dc464 100644
--- a/build-tools-internal/version.properties
+++ b/build-tools-internal/version.properties
@@ -48,8 +48,8 @@ jmh               = 1.26
 
 # test dependencies
 # when updating this version, also update :qa:evil-tests
-jimfs = 1.2
-jimfs_guava = 30.1-jre
+jimfs = 1.3.0
+jimfs_guava = 32.1.1-jre
 
 # test framework
 networknt_json_schema_validator = 1.0.48
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index ccb7b4dea36ee..7a894242f785b 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -481,6 +481,11 @@
             <sha256 value="721cb91842b46fa056847d104d5225c8b8e1e8b62263b993051e1e5a0137b7ec" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.errorprone" name="error_prone_annotations" version="2.18.0">
+         <artifact name="error_prone_annotations-2.18.0.jar">
+            <sha256 value="9e6814cb71816988a4fd1b07a993a8f21bb7058d522c162b1de849e19bea54ae" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.errorprone" name="error_prone_annotations" version="2.2.0">
          <artifact name="error_prone_annotations-2.2.0.jar">
             <sha256 value="6ebd22ca1b9d8ec06d41de8d64e0596981d9607b42035f9ed374f9de271a481a" origin="Generated by Gradle"/>
@@ -521,11 +526,6 @@
             <sha256 value="fc3aa363ad87223d1fbea584eee015a862150f6d34c71f24dc74088a635f08ef" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.guava" name="guava" version="30.1-jre">
-         <artifact name="guava-30.1-jre.jar">
-            <sha256 value="e6dd072f9d3fe02a4600688380bd422bdac184caf6fe2418cfdd0934f09432aa" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
       <component group="com.google.guava" name="guava" version="31.1-jre">
          <artifact name="guava-31.1-jre.jar">
             <sha256 value="a42edc9cab792e39fe39bb94f3fca655ed157ff87a8af78e1d6ba5b07c4a00ab" origin="Generated by Gradle"/>
@@ -536,6 +536,11 @@
             <sha256 value="bd7fa227591fb8509677d0d1122cf95158f3b8a9f45653f58281d879f6dc48c5" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.guava" name="guava" version="32.1.1-jre">
+         <artifact name="guava-32.1.1-jre.jar">
+            <sha256 value="91fbba37f1c8b251cf9ea9e7d3a369eb79eb1e6a5df1d4bbf483dd0380740281" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.guava" name="listenablefuture" version="9999.0-empty-to-avoid-conflict-with-guava">
          <artifact name="listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar">
             <sha256 value="b372a037d4230aa57fbeffdef30fd6123f9c0c2db85d0aced00c91b974f33f99" origin="Generated by Gradle"/>
@@ -606,9 +611,9 @@
             <sha256 value="21af30c92267bd6122c0e0b4d20cccb6641a37eaf956c6540ec471d584e64a7b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.jimfs" name="jimfs" version="1.2">
-         <artifact name="jimfs-1.2.jar">
-            <sha256 value="de16d5c8489729a8512f1a02fbd81f58f89249b72066987da4cc5c87ecb9f72d" origin="Generated by Gradle"/>
+      <component group="com.google.jimfs" name="jimfs" version="1.3.0">
+         <artifact name="jimfs-1.3.0.jar">
+            <sha256 value="82494408bb513f5512652e7b7f63d6f31f01eff57ce35c878644ffc2d25aee4f" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.google.oauth-client" name="google-oauth-client" version="1.34.1">
@@ -2886,6 +2891,11 @@
             <sha256 value="ff10785ac2a357ec5de9c293cb982a2cbb605c0309ea4cc1cb9b9bc6dbe7f3cb" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.checkerframework" name="checker-qual" version="3.33.0">
+         <artifact name="checker-qual-3.33.0.jar">
+            <sha256 value="e316255bbfcd9fe50d165314b85abb2b33cb2a66a93c491db648e498a82c2de1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.checkerframework" name="checker-qual" version="3.5.0">
          <artifact name="checker-qual-3.5.0.jar">
             <sha256 value="729990b3f18a95606fc2573836b6958bcdb44cb52bfbd1b7aa9c339cff35a5a4" origin="Generated by Gradle"/>
diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle
index 29eb7843101f5..ef040b002cd39 100644
--- a/qa/evil-tests/build.gradle
+++ b/qa/evil-tests/build.gradle
@@ -16,7 +16,7 @@ apply plugin: 'elasticsearch.internal-testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
-  testImplementation "com.google.jimfs:jimfs:1.2"
+  testImplementation "com.google.jimfs:jimfs:1.3.0"
   testImplementation "com.google.guava:guava:${versions.jimfs_guava}"
   testImplementation project(":test:framework")
   testImplementation project(':distribution:tools:plugin-cli')
diff --git a/x-pack/plugin/identity-provider/build.gradle b/x-pack/plugin/identity-provider/build.gradle
index 65b8e777e4c74..38ca5dfd344b8 100644
--- a/x-pack/plugin/identity-provider/build.gradle
+++ b/x-pack/plugin/identity-provider/build.gradle
@@ -48,7 +48,7 @@ dependencies {
   api "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
   api "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
   api "org.apache.httpcomponents:httpclient-cache:${versions.httpclient}"
-  runtimeOnly 'com.google.guava:guava:31.1-jre'
+  runtimeOnly 'com.google.guava:guava:32.0.1-jre'
   runtimeOnly 'com.google.guava:failureaccess:1.0.1'
   runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}"
 
@@ -256,7 +256,6 @@ tasks.named("thirdPartyAudit").configure {
     'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
     'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
     'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
-    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
     'com.google.common.hash.Striped64',
     'com.google.common.hash.Striped64$1',
     'com.google.common.hash.Striped64$Cell',
diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle
index fc5774ccbdb3a..0b7de9a0996ef 100644
--- a/x-pack/plugin/security/build.gradle
+++ b/x-pack/plugin/security/build.gradle
@@ -71,7 +71,7 @@ dependencies {
   api "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
   api "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
   api "org.apache.httpcomponents:httpclient-cache:${versions.httpclient}"
-  runtimeOnly 'com.google.guava:guava:31.1-jre'
+  runtimeOnly 'com.google.guava:guava:32.0.1-jre'
   runtimeOnly 'com.google.guava:failureaccess:1.0.1'
   runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}"
   runtimeOnly "joda-time:joda-time:2.10.10"
@@ -378,7 +378,6 @@ tasks.named("thirdPartyAudit").configure {
     'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
     'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
     'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
-    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
     'com.google.common.hash.Striped64',
     'com.google.common.hash.Striped64$1',
     'com.google.common.hash.Striped64$Cell',

From 9cda6d1e95a652b611d5c4b9e98205bafde4abc6 Mon Sep 17 00:00:00 2001
From: Rene Groeschke <rene@elastic.co>
Date: Mon, 18 Sep 2023 17:07:22 +0200
Subject: [PATCH 20/41] Restrict access to Legacy Gradle plugins (#99581)

This ensures new projects will not use legacy gradle plugins
---
 ...gConventionsPrecommitPluginFuncTest.groovy |   2 +
 ...acyYamlRestCompatTestPluginFuncTest.groovy |   2 +
 .../LegacyYamlRestTestPluginFuncTest.groovy   |   5 +
 .../internal/RestrictedBuildApiService.java   | 220 ++++++++++++++++++
 .../test/LegacyRestTestBasePlugin.java        |   9 +
 .../AbstractYamlRestCompatTestPlugin.java     |  12 +-
 .../fixtures/AbstractGradleFuncTest.groovy    |  24 +-
 ...rictedBuildServiceAwareGradleRunner.groovy |  17 ++
 ... BuildConfigurationAwareGradleRunner.java} |  20 +-
 9 files changed, 285 insertions(+), 26 deletions(-)
 create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
 create mode 100644 build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/RestrictedBuildServiceAwareGradleRunner.groovy
 rename build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/{ConfigurationCacheCompatibleAwareGradleRunner.java => BuildConfigurationAwareGradleRunner.java} (84%)

diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy
index 31ecffc07f63f..f6cbef21f5655 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy
@@ -172,6 +172,7 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl
 
     def "applies conventions on yaml-rest-test tests"() {
         given:
+        buildApiRestrictionsDisabled = true
         clazz(dir('src/yamlRestTest/java'), "org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase")
         buildFile << """
         apply plugin:'elasticsearch.legacy-yaml-rest-test'
@@ -211,6 +212,7 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl
     @Unroll
     def "applies conventions on #sourceSetName tests"() {
         given:
+        buildApiRestrictionsDisabled = pluginName.contains('legacy')
         clazz(dir("src/${sourceSetName}/java"), "org.elasticsearch.test.ESIntegTestCase")
         clazz(dir("src/${sourceSetName}/java"), "org.elasticsearch.test.rest.ESRestTestCase")
         buildFile << """
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy
index cf0cab3b952f3..74f5b32dff3cc 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy
@@ -33,7 +33,9 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         // 1. TestClustersPlugin not cc compatible due to listener registration
         // 2. RestIntegTestTask not cc compatible due to
         configurationCacheCompatible = false
+        buildApiRestrictionsDisabled = true
     }
+
     def "yamlRestTestVxCompatTest does nothing when there are no tests"() {
         given:
         subProject(":distribution:bwc:maintenance") << """
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy
index 599ad6d2df865..888c0cc83fc15 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy
@@ -17,6 +17,11 @@ import org.gradle.testkit.runner.TaskOutcome
 @IgnoreIf({ os.isWindows() })
 class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
+    def setup() {
+        buildApiRestrictionsDisabled = true
+    }
+
+
     def "yamlRestTest does nothing when there are no tests"() {
         given:
         // RestIntegTestTask not cc compatible due to
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
new file mode 100644
index 0000000000000..e9438eabadbb6
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
@@ -0,0 +1,220 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
+
+import org.elasticsearch.gradle.internal.test.LegacyRestTestBasePlugin;
+import org.gradle.api.GradleException;
+import org.gradle.api.Project;
+import org.gradle.api.provider.Property;
+import org.gradle.api.services.BuildService;
+import org.gradle.api.services.BuildServiceParameters;
+
+public abstract class RestrictedBuildApiService implements BuildService<RestrictedBuildApiService.Params> {
+
+    public static final String BUILD_API_RESTRICTIONS_SYS_PROPERTY = "org.elasticsearch.gradle.build-api-restriction.disabled";
+
+    private static ListMultimap<Class<?>, String> usageWhitelist = createLegacyRestTestBasePluginUsage();
+
+    private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsage() {
+        ListMultimap<Class<?>, String> map = ArrayListMultimap.create(1, 200);
+        map.put(LegacyRestTestBasePlugin.class, ":docs");
+        map.put(LegacyRestTestBasePlugin.class, ":distribution:docker");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:analysis-common");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:ingest-attachment");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:ingest-common");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:ingest-user-agent");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:kibana");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:lang-expression");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:lang-mustache");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:lang-painless");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:mapper-extras");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:parent-join");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:percolator");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:rank-eval");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:reindex");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:repository-s3");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:repository-url");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:runtime-fields-common");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:transport-netty4");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-icu");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-kuromoji");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-nori");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-phonetic");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-smartcn");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-stempel");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-ukrainian");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-azure-classic");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-ec2");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-gce");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:mapper-annotated-text");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:mapper-murmur3");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:repository-hdfs");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:store-smb");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:ccs-rolling-upgrade-remote-cluster");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:ccs-unavailable-clusters");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:logging-config");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:mixed-cluster");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:multi-cluster-search");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-http");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-disabled");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-plugins");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:system-indices");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:unconfigured-node-name");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:verify-version-constants");
+        map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-delayed-aggs");
+        map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-die-with-dignity");
+        map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-error-query");
+        map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-latency-simulating-directory");
+        map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-seek-tracking-directory");
+        map.put(LegacyRestTestBasePlugin.class, ":test:yaml-rest-runner");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin");
+        map.put(LegacyRestTestBasePlugin.class, ":distribution:archives:integ-test-zip");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:core");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:logstash");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-constant-keyword");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-unsigned-long");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-version");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:vector-tile");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:wildcard");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:kerberos-tests");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:mixed-tier-cluster");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:password-protected-keystore");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:reindex-tests-with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:repository-old-versions");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade-basic");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade-multi-cluster");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:runtime-fields:core-with-mapped");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:runtime-fields:core-with-search");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:saml-idp-tests");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:security-example-spi-extension");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:security-setup-password-tests");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:smoke-test-plugins");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:smoke-test-plugins-ssl");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:smoke-test-security-with-mustache");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:xpack-prefix-rest-compat");
+        map.put(LegacyRestTestBasePlugin.class, ":modules:ingest-geoip:qa:file-based-update");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-ec2:qa:amazon-ec2");
+        map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-gce:qa:gce");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-basic-license");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-full-license");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-restricted-trust");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:runtime-fields:with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:jira");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:pagerduty");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:slack");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:async-search:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:async-search:qa:security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:autoscaling:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:downgrade-to-basic-license");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:multi-cluster");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:non-compliant-license");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:restart");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:early-deprecation-rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:downsample:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:downsample:qa:with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:enrich:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:enrich:qa:rest-with-advanced-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:enrich:qa:rest-with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:ccs-rolling-upgrade");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:correctness");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:mixed-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:multi-cluster-with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:esql:qa:security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:esql:qa:server:multi-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:esql:qa:server:single-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:graph:qa:with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:identity-provider:qa:idp-rest-tests");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:multi-cluster");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:multi-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:basic-multi-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:disabled");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:ml-with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:multi-cluster-tests-with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:native-multi-node-tests");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:single-node-tests");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:repositories-metering-api:qa:s3");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:hdfs");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:minio");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:s3");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:url");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:operator-privileges-tests");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:profile");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:security-disabled");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:smoke-test-all-realms");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:tls-basic");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:shutdown:qa:multi-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:shutdown:qa:rolling-upgrade");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:multi-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:fs");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:s3");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:hdfs");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:minio");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:s3");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:multi-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:no-sql");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:single-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:security:with-ssl");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:security:without-ssl");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:mixed-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:multi-cluster-with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:multi-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:single-node");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:security:with-ssl");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:security:without-ssl");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:stack:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:text-structure:qa:text-structure-with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:multi-cluster-tests-with-security");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:multi-node-tests");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:single-node-tests");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:vector-tile:qa:multi-cluster");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:watcher:qa:rest");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:watcher:qa:with-monitoring");
+        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:watcher:qa:with-security");
+        return map;
+    }
+
+    public void failOnUsageRestriction(Class<?> aClass, Project project) {
+        if (getParameters().getDisabled().getOrElse(false)) {
+            return;
+        }
+        if (isSupported(aClass, project.getPath()) == false) {
+            throw new GradleException("Usage of deprecated " + aClass.getName() + " in " + project.getPath());
+        }
+    }
+
+    private boolean isSupported(Class<?> aClass, String path) {
+        return usageWhitelist.get(aClass).contains(path);
+    }
+
+    public abstract static class Params implements BuildServiceParameters {
+        public abstract Property<Boolean> getDisabled();
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java
index f270205111456..eacc5da6220ab 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin;
 import org.elasticsearch.gradle.internal.FixtureStop;
 import org.elasticsearch.gradle.internal.InternalTestClustersPlugin;
+import org.elasticsearch.gradle.internal.RestrictedBuildApiService;
 import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks;
 import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
@@ -22,12 +23,14 @@
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.bundling.Zip;
 
 import javax.inject.Inject;
 
+import static org.elasticsearch.gradle.internal.RestrictedBuildApiService.BUILD_API_RESTRICTIONS_SYS_PROPERTY;
 import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME;
 import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.EXPLODED_BUNDLE_PLUGIN_TASK_NAME;
 
@@ -52,6 +55,12 @@ public LegacyRestTestBasePlugin(ProviderFactory providerFactory) {
 
     @Override
     public void apply(Project project) {
+        Provider<RestrictedBuildApiService> serviceProvider = project.getGradle()
+            .getSharedServices()
+            .registerIfAbsent("restrictedBuildAPI", RestrictedBuildApiService.class, spec -> {
+                spec.getParameters().getDisabled().set(Boolean.getBoolean(BUILD_API_RESTRICTIONS_SYS_PROPERTY));
+            });
+        serviceProvider.get().failOnUsageRestriction(getClass(), project);
         project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
         project.getPluginManager().apply(ElasticsearchTestBasePlugin.class);
         project.getPluginManager().apply(InternalTestClustersPlugin.class);
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java
index addf4f02f7f99..63433928feb5a 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java
@@ -11,13 +11,12 @@
 import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin;
-import org.elasticsearch.gradle.internal.test.LegacyRestTestBasePlugin;
 import org.elasticsearch.gradle.internal.test.rest.CopyRestApiTask;
 import org.elasticsearch.gradle.internal.test.rest.CopyRestTestsTask;
 import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin;
 import org.elasticsearch.gradle.internal.test.rest.RestResourcesExtension;
 import org.elasticsearch.gradle.internal.test.rest.RestResourcesPlugin;
-import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
+import org.elasticsearch.gradle.test.YamlRestTestPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -77,20 +76,17 @@ public void apply(Project project) {
         final Path compatRestResourcesDir = Path.of("restResources").resolve("v" + COMPATIBLE_VERSION);
         final Path compatSpecsDir = compatRestResourcesDir.resolve("yamlSpecs");
         final Path compatTestsDir = compatRestResourcesDir.resolve("yamlTests");
-
+        project.getPluginManager().apply(getBasePlugin());
         project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
-        project.getPluginManager().apply(TestClustersPlugin.class);
-        project.getPluginManager().apply(LegacyRestTestBasePlugin.class);
         project.getPluginManager().apply(RestResourcesPlugin.class);
-        project.getPluginManager().apply(getBasePlugin());
 
         RestResourcesExtension extension = project.getExtensions().getByType(RestResourcesExtension.class);
 
         // create source set
         SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
         SourceSet yamlCompatTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
-        SourceSet yamlTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME);
-        GradleUtils.extendSourceSet(project, LegacyYamlRestTestPlugin.SOURCE_SET_NAME, SOURCE_SET_NAME);
+        SourceSet yamlTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.YAML_REST_TEST);
+        GradleUtils.extendSourceSet(project, YamlRestTestPlugin.YAML_REST_TEST, SOURCE_SET_NAME);
 
         // copy compatible rest specs
         Configuration bwcMinorConfig = project.getConfigurations().create(BWC_MINOR_CONFIG_NAME);
diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
index 87459e63a53c1..8218829fe017b 100644
--- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
+++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
@@ -9,7 +9,7 @@
 package org.elasticsearch.gradle.fixtures
 
 import org.apache.commons.io.FileUtils
-import org.elasticsearch.gradle.internal.test.ConfigurationCacheCompatibleAwareGradleRunner
+import org.elasticsearch.gradle.internal.test.BuildConfigurationAwareGradleRunner
 import org.elasticsearch.gradle.internal.test.InternalAwareGradleRunner
 import org.elasticsearch.gradle.internal.test.NormalizeOutputGradleRunner
 import org.elasticsearch.gradle.internal.test.TestResultExtension
@@ -39,7 +39,8 @@ abstract class AbstractGradleFuncTest extends Specification {
     File propertiesFile
     File projectDir
 
-    boolean configurationCacheCompatible = true
+    protected boolean configurationCacheCompatible = true
+    protected boolean buildApiRestrictionsDisabled = false
 
     def setup() {
         projectDir = testProjectDir.root
@@ -79,16 +80,17 @@ abstract class AbstractGradleFuncTest extends Specification {
 
     GradleRunner gradleRunner(File projectDir, Object... arguments) {
         return new NormalizeOutputGradleRunner(
-            new ConfigurationCacheCompatibleAwareGradleRunner(
+            new BuildConfigurationAwareGradleRunner(
                     new InternalAwareGradleRunner(
-                            GradleRunner.create()
-                                    .withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments()
-                                            .toString().indexOf("-agentlib:jdwp") > 0
-                                    )
-                                    .withProjectDir(projectDir)
-                                    .withPluginClasspath()
-                                    .forwardOutput()
-                    ), configurationCacheCompatible),
+                        GradleRunner.create()
+                                .withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments()
+                                        .toString().indexOf("-agentlib:jdwp") > 0
+                                )
+                                .withProjectDir(projectDir)
+                                .withPluginClasspath()
+                                .forwardOutput()
+            ), configurationCacheCompatible,
+                buildApiRestrictionsDisabled)
         ).withArguments(arguments.collect { it.toString() })
     }
 
diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/RestrictedBuildServiceAwareGradleRunner.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/RestrictedBuildServiceAwareGradleRunner.groovy
new file mode 100644
index 0000000000000..d8739b9b380cf
--- /dev/null
+++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/RestrictedBuildServiceAwareGradleRunner.groovy
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.fixtures
+
+import org.elasticsearch.gradle.internal.test.InternalAwareGradleRunner
+
+class RestrictedBuildServiceAwareGradleRunner {
+    RestrictedBuildServiceAwareGradleRunner(InternalAwareGradleRunner delegate) {
+
+    }
+}
diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/BuildConfigurationAwareGradleRunner.java
similarity index 84%
rename from build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java
rename to build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/BuildConfigurationAwareGradleRunner.java
index 74b92f2ab1389..1a2a7a564b4f1 100644
--- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java
+++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/BuildConfigurationAwareGradleRunner.java
@@ -23,15 +23,19 @@
 import java.util.Map;
 
 /**
- * A Gradle runner that delegates to another runner, optionally enabling the configuring cache parameter.
+ * A Gradle runner that delegates to another runner, optionally configuring
+ * - the configuring cache parameter
+ * - the internal restricted build api service
  */
-public class ConfigurationCacheCompatibleAwareGradleRunner extends GradleRunner {
-    private GradleRunner delegate;
-    private boolean ccCompatible;
+public class BuildConfigurationAwareGradleRunner extends GradleRunner {
+    private final GradleRunner delegate;
+    private final boolean ccCompatible;
+    private final boolean buildApiRestrictionsDisabled;
 
-    public ConfigurationCacheCompatibleAwareGradleRunner(GradleRunner delegate, boolean ccCompatible) {
+    public BuildConfigurationAwareGradleRunner(GradleRunner delegate, boolean ccCompatible, boolean buildApiRestrictionsDisabled) {
         this.delegate = delegate;
         this.ccCompatible = ccCompatible;
+        this.buildApiRestrictionsDisabled = buildApiRestrictionsDisabled;
     }
 
     @Override
@@ -76,11 +80,13 @@ public List<String> getArguments() {
 
     @Override
     public GradleRunner withArguments(List<String> arguments) {
-        List<String> effectiveArgs = arguments;
+        List<String> effectiveArgs = new ArrayList<>(arguments);
         if (ccCompatible) {
-            effectiveArgs = new ArrayList<>(arguments);
             effectiveArgs.add("--configuration-cache");
         }
+        if (buildApiRestrictionsDisabled) {
+            effectiveArgs.add("-Dorg.elasticsearch.gradle.build-api-restriction.disabled=" + buildApiRestrictionsDisabled);
+        }
         delegate.withArguments(effectiveArgs);
         return this;
     }

From d58b9ea87d23c51a801a45fdb13d38d9f8d2539c Mon Sep 17 00:00:00 2001
From: gheorghepucea <91881042+gheorghepucea@users.noreply.github.com>
Date: Mon, 18 Sep 2023 17:29:20 +0200
Subject: [PATCH 21/41] Added esql ends_with implementation (#99613)

Added an implementation for `ends_with` function in esql.  `ends_with` -
Returns a boolean that indicates whether a keyword string ends with
another string. Also made sure that the docs look alright:

<img width="1677" alt="Screenshot 2023-09-16 at 18 10 46"
src="https://github.com/elastic/elasticsearch/assets/91881042/eccd81e1-40a2-4a66-a514-cf3e4205f9da">
---
 docs/reference/esql/esql-functions.asciidoc   |   2 +
 .../esql/functions/ends_with.asciidoc         |  20 +++
 .../esql/functions/signature/ends_with.svg    |   1 +
 .../esql/functions/types/ends_with.asciidoc   |   5 +
 .../src/main/resources/boolean.csv-spec       |  15 ++
 .../src/main/resources/show.csv-spec          |   1 +
 .../src/main/resources/string.csv-spec        |  71 ++++++++++
 .../scalar/string/EndsWithEvaluator.java      |  93 +++++++++++++
 .../function/EsqlFunctionRegistry.java        |   4 +-
 .../function/scalar/string/EndsWith.java      | 105 ++++++++++++++
 .../xpack/esql/io/stream/PlanNamedTypes.java  |  13 ++
 .../function/scalar/string/EndsWithTests.java | 130 ++++++++++++++++++
 12 files changed, 459 insertions(+), 1 deletion(-)
 create mode 100644 docs/reference/esql/functions/ends_with.asciidoc
 create mode 100644 docs/reference/esql/functions/signature/ends_with.svg
 create mode 100644 docs/reference/esql/functions/types/ends_with.asciidoc
 create mode 100644 x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java

diff --git a/docs/reference/esql/esql-functions.asciidoc b/docs/reference/esql/esql-functions.asciidoc
index 1e29d312906ae..9d656760fe12e 100644
--- a/docs/reference/esql/esql-functions.asciidoc
+++ b/docs/reference/esql/esql-functions.asciidoc
@@ -26,6 +26,7 @@ these functions:
 * <<esql-date_parse>>
 * <<esql-date_trunc>>
 * <<esql-e>>
+* <<esql-ends_with>>
 * <<esql-floor>>
 * <<esql-greatest>>
 * <<esql-is_finite>>
@@ -89,6 +90,7 @@ include::functions/date_format.asciidoc[]
 include::functions/date_parse.asciidoc[]
 include::functions/date_trunc.asciidoc[]
 include::functions/e.asciidoc[]
+include::functions/ends_with.asciidoc[]
 include::functions/floor.asciidoc[]
 include::functions/greatest.asciidoc[]
 include::functions/is_finite.asciidoc[]
diff --git a/docs/reference/esql/functions/ends_with.asciidoc b/docs/reference/esql/functions/ends_with.asciidoc
new file mode 100644
index 0000000000000..baa96a378ab37
--- /dev/null
+++ b/docs/reference/esql/functions/ends_with.asciidoc
@@ -0,0 +1,20 @@
+[[esql-ends_with]]
+=== `ENDS_WITH`
+[.text-center]
+image::esql/functions/signature/ends_with.svg[Embedded,opts=inline]
+
+Returns a boolean that indicates whether a keyword string ends with another
+string:
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/string.csv-spec[tag=endsWith]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/string.csv-spec[tag=endsWith-result]
+|===
+
+Supported types:
+
+include::types/ends_with.asciidoc[]
diff --git a/docs/reference/esql/functions/signature/ends_with.svg b/docs/reference/esql/functions/signature/ends_with.svg
new file mode 100644
index 0000000000000..bf2cb47ed0be0
--- /dev/null
+++ b/docs/reference/esql/functions/signature/ends_with.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="420" height="46" viewbox="0 0 420 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m128 0h10m32 0h10m68 0h10m32 0h10m68 0h10m32 0h5"/><rect class="s" x="5" y="5" width="128" height="36"/><text class="k" x="15" y="31">ENDS_WITH</text><rect class="s" x="143" y="5" width="32" height="36" rx="7"/><text class="syn" x="153" y="31">(</text><rect class="s" x="185" y="5" width="68" height="36" rx="7"/><text class="k" x="195" y="31">arg1</text><rect class="s" x="263" y="5" width="32" height="36" rx="7"/><text class="syn" x="273" y="31">,</text><rect class="s" x="305" y="5" width="68" height="36" rx="7"/><text class="k" x="315" y="31">arg2</text><rect class="s" x="383" y="5" width="32" height="36" rx="7"/><text class="syn" x="393" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/types/ends_with.asciidoc b/docs/reference/esql/functions/types/ends_with.asciidoc
new file mode 100644
index 0000000000000..6c406b80c0cad
--- /dev/null
+++ b/docs/reference/esql/functions/types/ends_with.asciidoc
@@ -0,0 +1,5 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+arg1 | arg2 | result
+keyword | keyword | boolean
+|===
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/boolean.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/boolean.csv-spec
index ec94d0a849043..7426f07042962 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/boolean.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/boolean.csv-spec
@@ -47,6 +47,21 @@ avg(salary):double | always_false:boolean
  48353.72222222222 | false
 ;
 
+statsWithEndsWithAlwaysTrue
+from employees | where first_name is not null | eval always_true = ends_with(first_name, "") | stats avg(salary) by always_true;
+
+avg(salary):double | always_true:boolean
+ 48353.72222222222 | true
+;
+
+statsWithEndsWithAlwaysFalse
+from employees | where first_name is not null | eval always_false = ends_with(first_name, "noneendswiththis") | stats avg(salary) by always_false;
+
+avg(salary):double | always_false:boolean
+ 48353.72222222222 | false
+;
+
+
 in
 from employees | keep emp_no, is_rehired, still_hired | where is_rehired in (still_hired, true) | where is_rehired != still_hired;
 ignoreOrder:true
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec
index cea3490f156bc..360ef3b0a70b8 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec
@@ -30,6 +30,7 @@ date_format              |date_format(arg1, arg2)
 date_parse               |date_parse(arg1, arg2)
 date_trunc               |date_trunc(arg1, arg2)
 e                        |e()
+ends_with                |ends_with(arg1, arg2)
 floor                    |floor(n)
 greatest                 |greatest(first, rest...)
 is_finite                |is_finite(arg1)
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
index 357f6369dca73..41d70c65cd5f9 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
@@ -760,3 +760,74 @@ Bamford           |ord
 Bernatsky         |sky
 // end::right-result[]
 ;
+
+endsWithConstant
+from employees | sort emp_no | limit 10 | eval f_a = ends_with(first_name, "a") | keep emp_no, first_name, f_a;
+
+emp_no:integer | first_name:keyword  | f_a:boolean
+10001 | Georgi    | false
+10002 | Bezalel   | false
+10003 | Parto     | false
+10004 | Chirstian | false
+10005 | Kyoichi   | false
+10006 | Anneke    | false
+10007 | Tzvetan   | false
+10008 | Saniya    | true
+10009 | Sumant    | false
+10010 | Duangkaew | false
+;
+
+endsWithSequence
+from employees | sort emp_no | limit 10 | eval f_a = ends_with(first_name, "an") | keep emp_no, first_name, f_a;
+
+emp_no:integer | first_name:keyword  | f_a:boolean
+10001 | Georgi    | false
+10002 | Bezalel   | false
+10003 | Parto     | false
+10004 | Chirstian | true
+10005 | Kyoichi   | false
+10006 | Anneke    | false
+10007 | Tzvetan   | true
+10008 | Saniya    | false
+10009 | Sumant    | false
+10010 | Duangkaew | false
+;
+
+endsWithExpression
+from employees | sort emp_no | limit 10
+| eval last_name_last_letter = right(last_name, 1)
+| eval same_last_letters  = ends_with(first_name, last_name_last_letter)
+| keep emp_no, first_name, last_name, same_last_letters;
+
+emp_no:integer | first_name:keyword  | last_name:keyword | same_last_letters:boolean
+10001 | Georgi    | Facello   | false
+10002 | Bezalel   | Simmel    | true
+10003 | Parto     | Bamford   | false
+10004 | Chirstian | Koblick   | false
+10005 | Kyoichi   | Maliniak  | false
+10006 | Anneke    | Preusig   | false
+10007 | Tzvetan   | Zielinski | false
+10008 | Saniya    | Kalloufi  | false
+10009 | Sumant    | Peac      | false
+10010 | Duangkaew | Piveteau  | false
+;
+
+docsEndsWith
+// tag::endsWith[]
+FROM employees
+| KEEP last_name
+| EVAL ln_E = ENDS_WITH(last_name, "d")
+// end::endsWith[]
+| SORT last_name ASC
+| LIMIT 5
+;
+
+// tag::endsWith-result[]
+last_name:keyword | ln_E:boolean
+Awdeh          |false
+Azuma          |false
+Baek           |false
+Bamford        |true
+Bernatsky      |false
+// end::endsWith-result[]
+;
diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java
new file mode 100644
index 0000000000000..4206dc8241058
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java
@@ -0,0 +1,93 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import java.lang.Override;
+import java.lang.String;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link EndsWith}.
+ * This class is generated. Do not edit it.
+ */
+public final class EndsWithEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final EvalOperator.ExpressionEvaluator str;
+
+  private final EvalOperator.ExpressionEvaluator suffix;
+
+  private final DriverContext driverContext;
+
+  public EndsWithEvaluator(EvalOperator.ExpressionEvaluator str,
+      EvalOperator.ExpressionEvaluator suffix, DriverContext driverContext) {
+    this.str = str;
+    this.suffix = suffix;
+    this.driverContext = driverContext;
+  }
+
+  @Override
+  public Block eval(Page page) {
+    Block strUncastBlock = str.eval(page);
+    if (strUncastBlock.areAllValuesNull()) {
+      return Block.constantNullBlock(page.getPositionCount());
+    }
+    BytesRefBlock strBlock = (BytesRefBlock) strUncastBlock;
+    Block suffixUncastBlock = suffix.eval(page);
+    if (suffixUncastBlock.areAllValuesNull()) {
+      return Block.constantNullBlock(page.getPositionCount());
+    }
+    BytesRefBlock suffixBlock = (BytesRefBlock) suffixUncastBlock;
+    BytesRefVector strVector = strBlock.asVector();
+    if (strVector == null) {
+      return eval(page.getPositionCount(), strBlock, suffixBlock);
+    }
+    BytesRefVector suffixVector = suffixBlock.asVector();
+    if (suffixVector == null) {
+      return eval(page.getPositionCount(), strBlock, suffixBlock);
+    }
+    return eval(page.getPositionCount(), strVector, suffixVector).asBlock();
+  }
+
+  public BooleanBlock eval(int positionCount, BytesRefBlock strBlock, BytesRefBlock suffixBlock) {
+    BooleanBlock.Builder result = BooleanBlock.newBlockBuilder(positionCount);
+    BytesRef strScratch = new BytesRef();
+    BytesRef suffixScratch = new BytesRef();
+    position: for (int p = 0; p < positionCount; p++) {
+      if (strBlock.isNull(p) || strBlock.getValueCount(p) != 1) {
+        result.appendNull();
+        continue position;
+      }
+      if (suffixBlock.isNull(p) || suffixBlock.getValueCount(p) != 1) {
+        result.appendNull();
+        continue position;
+      }
+      result.appendBoolean(EndsWith.process(strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), suffixBlock.getBytesRef(suffixBlock.getFirstValueIndex(p), suffixScratch)));
+    }
+    return result.build();
+  }
+
+  public BooleanVector eval(int positionCount, BytesRefVector strVector,
+      BytesRefVector suffixVector) {
+    BooleanVector.Builder result = BooleanVector.newVectorBuilder(positionCount);
+    BytesRef strScratch = new BytesRef();
+    BytesRef suffixScratch = new BytesRef();
+    position: for (int p = 0; p < positionCount; p++) {
+      result.appendBoolean(EndsWith.process(strVector.getBytesRef(p, strScratch), suffixVector.getBytesRef(p, suffixScratch)));
+    }
+    return result.build();
+  }
+
+  @Override
+  public String toString() {
+    return "EndsWithEvaluator[" + "str=" + str + ", suffix=" + suffix + "]";
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
index d4029fd8ac4f0..eecb2821ca135 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -70,6 +70,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum;
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.EndsWith;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length;
@@ -145,7 +146,8 @@ private FunctionDefinition[][] functions() {
                 def(Trim.class, Trim::new, "trim"),
                 def(Left.class, Left::new, "left"),
                 def(Right.class, Right::new, "right"),
-                def(StartsWith.class, StartsWith::new, "starts_with") },
+                def(StartsWith.class, StartsWith::new, "starts_with"),
+                def(EndsWith.class, EndsWith::new, "ends_with") },
             // date
             new FunctionDefinition[] {
                 def(DateExtract.class, DateExtract::new, "date_extract"),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java
new file mode 100644
index 0000000000000..367d5323f28b8
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.ann.Evaluator;
+import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
+import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.FIRST;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.SECOND;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString;
+
+public class EndsWith extends ScalarFunction implements EvaluatorMapper {
+
+    private final Expression str;
+    private final Expression suffix;
+
+    public EndsWith(Source source, Expression str, Expression suffix) {
+        super(source, Arrays.asList(str, suffix));
+        this.str = str;
+        this.suffix = suffix;
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.BOOLEAN;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution resolution = isString(str, sourceText(), FIRST);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+        return isString(suffix, sourceText(), SECOND);
+    }
+
+    @Override
+    public boolean foldable() {
+        return str.foldable() && suffix.foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return EvaluatorMapper.super.fold();
+    }
+
+    @Evaluator
+    static boolean process(BytesRef str, BytesRef suffix) {
+        if (str.length < suffix.length) {
+            return false;
+        }
+        return Arrays.equals(
+            str.bytes,
+            str.offset + str.length - suffix.length,
+            str.offset + str.length,
+            suffix.bytes,
+            suffix.offset,
+            suffix.offset + suffix.length
+        );
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new EndsWith(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, EndsWith::new, str, suffix);
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        throw new UnsupportedOperationException("functions do not support scripting");
+    }
+
+    @Override
+    public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
+        var strEval = toEvaluator.apply(str);
+        var suffixEval = toEvaluator.apply(suffix);
+        return dvrCtx -> new EndsWithEvaluator(strEval.get(dvrCtx), suffixEval.get(dvrCtx), dvrCtx);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index edc8bcacda9d4..91a3dd62232ec 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -88,6 +88,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum;
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.EndsWith;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length;
@@ -345,6 +346,7 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
             of(ScalarFunction.class, Round.class, PlanNamedTypes::writeRound, PlanNamedTypes::readRound),
             of(ScalarFunction.class, Pow.class, PlanNamedTypes::writePow, PlanNamedTypes::readPow),
             of(ScalarFunction.class, StartsWith.class, PlanNamedTypes::writeStartsWith, PlanNamedTypes::readStartsWith),
+            of(ScalarFunction.class, EndsWith.class, PlanNamedTypes::writeEndsWith, PlanNamedTypes::readEndsWith),
             of(ScalarFunction.class, Substring.class, PlanNamedTypes::writeSubstring, PlanNamedTypes::readSubstring),
             of(ScalarFunction.class, Left.class, PlanNamedTypes::writeLeft, PlanNamedTypes::readLeft),
             of(ScalarFunction.class, Right.class, PlanNamedTypes::writeRight, PlanNamedTypes::readRight),
@@ -1283,6 +1285,17 @@ static void writeStartsWith(PlanStreamOutput out, StartsWith startsWith) throws
         out.writeExpression(fields.get(1));
     }
 
+    static EndsWith readEndsWith(PlanStreamInput in) throws IOException {
+        return new EndsWith(Source.EMPTY, in.readExpression(), in.readExpression());
+    }
+
+    static void writeEndsWith(PlanStreamOutput out, EndsWith endsWith) throws IOException {
+        List<Expression> fields = endsWith.children();
+        assert fields.size() == 2;
+        out.writeExpression(fields.get(0));
+        out.writeExpression(fields.get(1));
+    }
+
     static Substring readSubstring(PlanStreamInput in) throws IOException {
         return new Substring(Source.EMPTY, in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class));
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
new file mode 100644
index 0000000000000..63e70b6612470
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+import org.hamcrest.Matcher;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class EndsWithTests extends AbstractScalarFunctionTestCase {
+    public EndsWithTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        List<TestCaseSupplier> suppliers = new LinkedList<>();
+        suppliers.add(new TestCaseSupplier("ends_with empty suffix", () -> {
+            String str = randomAlphaOfLength(5);
+            String suffix = "";
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataTypes.KEYWORD, "suffix")
+                ),
+                "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
+                DataTypes.BOOLEAN,
+                equalTo(str.endsWith(suffix))
+            );
+        }));
+        suppliers.add(new TestCaseSupplier("ends_with empty str", () -> {
+            String str = "";
+            String suffix = randomAlphaOfLength(5);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataTypes.KEYWORD, "suffix")
+                ),
+                "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
+                DataTypes.BOOLEAN,
+                equalTo(str.endsWith(suffix))
+            );
+        }));
+        suppliers.add(new TestCaseSupplier("ends_with one char suffix", () -> {
+            String str = randomAlphaOfLength(5);
+            String suffix = randomAlphaOfLength(1);
+            str = str + suffix;
+
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataTypes.KEYWORD, "suffix")
+                ),
+                "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
+                DataTypes.BOOLEAN,
+                equalTo(str.endsWith(suffix))
+            );
+        }));
+        suppliers.add(new TestCaseSupplier("ends_with no match suffix", () -> {
+            String str = randomAlphaOfLength(5);
+            String suffix = "no_match_suffix";
+            str = suffix + str;
+
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataTypes.KEYWORD, "suffix")
+                ),
+                "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
+                DataTypes.BOOLEAN,
+                equalTo(str.endsWith(suffix))
+            );
+        }));
+        suppliers.add(new TestCaseSupplier("ends_with randomized test", () -> {
+            String str = randomRealisticUnicodeOfLength(5);
+            String suffix = randomRealisticUnicodeOfLength(5);
+            str = str + suffix;
+
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataTypes.KEYWORD, "suffix")
+                ),
+                "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
+                DataTypes.BOOLEAN,
+                equalTo(str.endsWith(suffix))
+            );
+        }));
+        return parameterSuppliersFromTypedData(suppliers);
+    }
+
+    @Override
+    protected DataType expectedType(List<DataType> argTypes) {
+        return DataTypes.BOOLEAN;
+    }
+
+    private Matcher<Object> resultsMatcher(List<TestCaseSupplier.TypedData> typedData) {
+        String str = ((BytesRef) typedData.get(0).data()).utf8ToString();
+        String prefix = ((BytesRef) typedData.get(1).data()).utf8ToString();
+        return equalTo(str.endsWith(prefix));
+    }
+
+    @Override
+    protected List<ArgumentSpec> argSpec() {
+        return List.of(required(strings()), required(strings()));
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new EndsWith(source, args.get(0), args.get(1));
+    }
+}

From 1a14bc7a06e5b168cbccf28bd3bf450f03e5e236 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 18 Sep 2023 11:33:39 -0400
Subject: [PATCH 22/41] ESQL: Prevent topn from using too much memory (#99611)

This prevents topn operations from using too much memory by hooking them
into circuit breaking framework. It builds on the work done in
https://github.com/elastic/elasticsearch/pull/99316 that moved all topn
storage to byte arrays by adding circuit breaking to process of growing
the underlying byte array.
---
 .../compute/operator/TopNBenchmark.java       |  11 +
 .../common/util/MockBigArrays.java            |  17 +-
 .../operator/topn/KeyExtractorForBoolean.java |  16 +-
 .../topn/KeyExtractorForBytesRef.java         |  16 +-
 .../operator/topn/KeyExtractorForDouble.java  |  16 +-
 .../operator/topn/KeyExtractorForInt.java     |  16 +-
 .../operator/topn/KeyExtractorForLong.java    |  16 +-
 .../topn/ValueExtractorForBoolean.java        |  10 +-
 .../topn/ValueExtractorForBytesRef.java       |  10 +-
 .../topn/ValueExtractorForDouble.java         |  10 +-
 .../operator/topn/ValueExtractorForInt.java   |  10 +-
 .../operator/topn/ValueExtractorForLong.java  |  10 +-
 .../operator/BreakingBytesRefBuilder.java     | 149 +++++++++++
 .../topn/DefaultSortableTopNEncoder.java      |   4 +-
 .../topn/DefaultUnsortableTopNEncoder.java    |  14 +-
 .../operator/topn/FixedLengthTopNEncoder.java |   4 +-
 .../compute/operator/topn/KeyExtractor.java   |   6 +-
 .../operator/topn/KeyExtractorForNull.java    |   4 +-
 .../operator/topn/SortableTopNEncoder.java    |  10 +-
 .../compute/operator/topn/TopNEncoder.java    |  12 +-
 .../compute/operator/topn/TopNOperator.java   | 245 +++++++++++-------
 .../operator/topn/UTF8TopNEncoder.java        |   4 +-
 .../compute/operator/topn/ValueExtractor.java |   6 +-
 .../operator/topn/ValueExtractorForDoc.java   |   4 +-
 .../operator/topn/ValueExtractorForNull.java  |   4 +-
 .../operator/topn/VersionTopNEncoder.java     |   4 +-
 .../operator/topn/X-KeyExtractor.java.st      |  16 +-
 .../operator/topn/X-ValueExtractor.java.st    |  10 +-
 .../compute/operator/AnyOperatorTestCase.java |   8 +-
 .../BreakingBytesRefBuilderTests.java         | 148 +++++++++++
 .../compute/operator/OperatorTestCase.java    |  19 +-
 .../DefaultUnsortableTopNEncoderTests.java    |   6 +-
 .../compute/operator/topn/ExtractorTests.java |  20 +-
 .../operator/topn/TopNEncoderTests.java       |  22 +-
 .../operator/topn/TopNOperatorTests.java      |  76 +++++-
 .../compute/operator/topn/TopNRowTests.java   |  29 ++-
 .../elasticsearch/xpack/esql/CsvTests.java    |   9 +-
 37 files changed, 738 insertions(+), 253 deletions(-)
 create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java
 create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java
index f4b5397e55d39..9ef4eef2a6924 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java
@@ -9,6 +9,9 @@
 package org.elasticsearch.benchmark.compute.operator;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
@@ -20,6 +23,8 @@
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.compute.operator.topn.TopNEncoder;
 import org.elasticsearch.compute.operator.topn.TopNOperator;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -96,7 +101,13 @@ private static Operator operator(String data, int topCount) {
             case LONGS_AND_BYTES_REFS -> List.of(TopNEncoder.DEFAULT_SORTABLE, TopNEncoder.UTF8);
             default -> throw new IllegalArgumentException("unsupported data type [" + data + "]");
         };
+        CircuitBreakerService breakerService = new HierarchyCircuitBreakerService(
+            Settings.EMPTY,
+            List.of(),
+            ClusterSettings.createBuiltInClusterSettings()
+        );
         return new TopNOperator(
+            breakerService.getBreaker(CircuitBreaker.REQUEST),
             topCount,
             elementTypes,
             encoders,
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
index 5c1d395ed28e5..02e8cfd7f16fc 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
@@ -675,9 +675,15 @@ public LimitedBreaker(String name, ByteSizeValue max) {
 
         @Override
         public void addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException {
-            long total = used.addAndGet(bytes);
-            if (total > max.getBytes()) {
-                throw new CircuitBreakingException(ERROR_MESSAGE, bytes, max.getBytes(), Durability.TRANSIENT);
+            while (true) {
+                long old = used.get();
+                long total = old + bytes;
+                if (total > max.getBytes()) {
+                    throw new CircuitBreakingException(ERROR_MESSAGE, bytes, max.getBytes(), Durability.TRANSIENT);
+                }
+                if (used.compareAndSet(old, total)) {
+                    break;
+                }
             }
         }
 
@@ -685,5 +691,10 @@ public void addEstimateBytesAndMaybeBreak(long bytes, String label) throws Circu
         public void addWithoutBreaking(long bytes) {
             used.addAndGet(bytes);
         }
+
+        @Override
+        public long getUsed() {
+            return used.get();
+        }
     }
 }
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForBoolean.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForBoolean.java
index 0f7a4e109af75..0bdc5ac620eb0 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForBoolean.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForBoolean.java
@@ -7,10 +7,10 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class KeyExtractorForBoolean implements KeyExtractor {
     static KeyExtractorForBoolean extractorFor(TopNEncoder encoder, boolean ascending, byte nul, byte nonNul, BooleanBlock block) {
@@ -37,13 +37,13 @@ static KeyExtractorForBoolean extractorFor(TopNEncoder encoder, boolean ascendin
         this.nonNul = nonNul;
     }
 
-    protected final int nonNul(BytesRefBuilder key, boolean value) {
+    protected final int nonNul(BreakingBytesRefBuilder key, boolean value) {
         key.append(nonNul);
         TopNEncoder.DEFAULT_SORTABLE.encodeBoolean(value, key);
         return Byte.BYTES + 1;
     }
 
-    protected final int nul(BytesRefBuilder key) {
+    protected final int nul(BreakingBytesRefBuilder key) {
         key.append(nul);
         return 1;
     }
@@ -57,7 +57,7 @@ static class ForVector extends KeyExtractorForBoolean {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             return nonNul(key, vector.getBoolean(position));
         }
     }
@@ -71,7 +71,7 @@ static class MinForAscending extends KeyExtractorForBoolean {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -88,7 +88,7 @@ static class MaxForAscending extends KeyExtractorForBoolean {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -105,7 +105,7 @@ static class MinForUnordered extends KeyExtractorForBoolean {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
@@ -130,7 +130,7 @@ static class MaxForUnordered extends KeyExtractorForBoolean {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForBytesRef.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForBytesRef.java
index d9d8d3878817e..accce46f38e30 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForBytesRef.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForBytesRef.java
@@ -8,10 +8,10 @@
 package org.elasticsearch.compute.operator.topn;
 
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class KeyExtractorForBytesRef implements KeyExtractor {
     static KeyExtractorForBytesRef extractorFor(TopNEncoder encoder, boolean ascending, byte nul, byte nonNul, BytesRefBlock block) {
@@ -40,12 +40,12 @@ static KeyExtractorForBytesRef extractorFor(TopNEncoder encoder, boolean ascendi
         this.nonNul = nonNul;
     }
 
-    protected final int nonNul(BytesRefBuilder key, BytesRef value) {
+    protected final int nonNul(BreakingBytesRefBuilder key, BytesRef value) {
         key.append(nonNul);
         return encoder.encodeBytesRef(value, key) + 1;
     }
 
-    protected final int nul(BytesRefBuilder key) {
+    protected final int nul(BreakingBytesRefBuilder key) {
         key.append(nul);
         return 1;
     }
@@ -59,7 +59,7 @@ static class ForVector extends KeyExtractorForBytesRef {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             return nonNul(key, vector.getBytesRef(position, scratch));
         }
     }
@@ -73,7 +73,7 @@ static class MinForAscending extends KeyExtractorForBytesRef {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -90,7 +90,7 @@ static class MaxForAscending extends KeyExtractorForBytesRef {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -109,7 +109,7 @@ static class MinForUnordered extends KeyExtractorForBytesRef {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
@@ -140,7 +140,7 @@ static class MaxForUnordered extends KeyExtractorForBytesRef {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForDouble.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForDouble.java
index 8d8458d33ab47..2f2968da16d83 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForDouble.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForDouble.java
@@ -7,10 +7,10 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.DoubleVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class KeyExtractorForDouble implements KeyExtractor {
     static KeyExtractorForDouble extractorFor(TopNEncoder encoder, boolean ascending, byte nul, byte nonNul, DoubleBlock block) {
@@ -37,13 +37,13 @@ static KeyExtractorForDouble extractorFor(TopNEncoder encoder, boolean ascending
         this.nonNul = nonNul;
     }
 
-    protected final int nonNul(BytesRefBuilder key, double value) {
+    protected final int nonNul(BreakingBytesRefBuilder key, double value) {
         key.append(nonNul);
         TopNEncoder.DEFAULT_SORTABLE.encodeDouble(value, key);
         return Double.BYTES + 1;
     }
 
-    protected final int nul(BytesRefBuilder key) {
+    protected final int nul(BreakingBytesRefBuilder key) {
         key.append(nul);
         return 1;
     }
@@ -57,7 +57,7 @@ static class ForVector extends KeyExtractorForDouble {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             return nonNul(key, vector.getDouble(position));
         }
     }
@@ -71,7 +71,7 @@ static class MinForAscending extends KeyExtractorForDouble {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -88,7 +88,7 @@ static class MaxForAscending extends KeyExtractorForDouble {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -105,7 +105,7 @@ static class MinForUnordered extends KeyExtractorForDouble {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
@@ -129,7 +129,7 @@ static class MaxForUnordered extends KeyExtractorForDouble {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForInt.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForInt.java
index 9c20f53689b0a..400c43168277d 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForInt.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForInt.java
@@ -7,10 +7,10 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class KeyExtractorForInt implements KeyExtractor {
     static KeyExtractorForInt extractorFor(TopNEncoder encoder, boolean ascending, byte nul, byte nonNul, IntBlock block) {
@@ -37,13 +37,13 @@ static KeyExtractorForInt extractorFor(TopNEncoder encoder, boolean ascending, b
         this.nonNul = nonNul;
     }
 
-    protected final int nonNul(BytesRefBuilder key, int value) {
+    protected final int nonNul(BreakingBytesRefBuilder key, int value) {
         key.append(nonNul);
         TopNEncoder.DEFAULT_SORTABLE.encodeInt(value, key);
         return Integer.BYTES + 1;
     }
 
-    protected final int nul(BytesRefBuilder key) {
+    protected final int nul(BreakingBytesRefBuilder key) {
         key.append(nul);
         return 1;
     }
@@ -57,7 +57,7 @@ static class ForVector extends KeyExtractorForInt {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             return nonNul(key, vector.getInt(position));
         }
     }
@@ -71,7 +71,7 @@ static class MinForAscending extends KeyExtractorForInt {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -88,7 +88,7 @@ static class MaxForAscending extends KeyExtractorForInt {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -105,7 +105,7 @@ static class MinForUnordered extends KeyExtractorForInt {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
@@ -129,7 +129,7 @@ static class MaxForUnordered extends KeyExtractorForInt {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForLong.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForLong.java
index 5ad6c8d9602a8..843efdd95471f 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForLong.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForLong.java
@@ -7,10 +7,10 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class KeyExtractorForLong implements KeyExtractor {
     static KeyExtractorForLong extractorFor(TopNEncoder encoder, boolean ascending, byte nul, byte nonNul, LongBlock block) {
@@ -37,13 +37,13 @@ static KeyExtractorForLong extractorFor(TopNEncoder encoder, boolean ascending,
         this.nonNul = nonNul;
     }
 
-    protected final int nonNul(BytesRefBuilder key, long value) {
+    protected final int nonNul(BreakingBytesRefBuilder key, long value) {
         key.append(nonNul);
         TopNEncoder.DEFAULT_SORTABLE.encodeLong(value, key);
         return Long.BYTES + 1;
     }
 
-    protected final int nul(BytesRefBuilder key) {
+    protected final int nul(BreakingBytesRefBuilder key) {
         key.append(nul);
         return 1;
     }
@@ -57,7 +57,7 @@ static class ForVector extends KeyExtractorForLong {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             return nonNul(key, vector.getLong(position));
         }
     }
@@ -71,7 +71,7 @@ static class MinForAscending extends KeyExtractorForLong {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -88,7 +88,7 @@ static class MaxForAscending extends KeyExtractorForLong {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -105,7 +105,7 @@ static class MinForUnordered extends KeyExtractorForLong {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
@@ -129,7 +129,7 @@ static class MaxForUnordered extends KeyExtractorForLong {
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForBoolean.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForBoolean.java
index 0136c795746d0..b13dd3ce7f2b0 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForBoolean.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForBoolean.java
@@ -7,9 +7,9 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class ValueExtractorForBoolean implements ValueExtractor {
     static ValueExtractorForBoolean extractorFor(TopNEncoder encoder, boolean inKey, BooleanBlock block) {
@@ -27,11 +27,11 @@ static ValueExtractorForBoolean extractorFor(TopNEncoder encoder, boolean inKey,
         this.inKey = inKey;
     }
 
-    protected final void writeCount(BytesRefBuilder values, int count) {
+    protected final void writeCount(BreakingBytesRefBuilder values, int count) {
         TopNEncoder.DEFAULT_UNSORTABLE.encodeVInt(count, values);
     }
 
-    protected final void actualWriteValue(BytesRefBuilder values, boolean value) {
+    protected final void actualWriteValue(BreakingBytesRefBuilder values, boolean value) {
         TopNEncoder.DEFAULT_UNSORTABLE.encodeBoolean(value, values);
     }
 
@@ -44,7 +44,7 @@ static class ForVector extends ValueExtractorForBoolean {
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             writeCount(values, 1);
             if (inKey) {
                 // will read results from the key
@@ -63,7 +63,7 @@ static class ForBlock extends ValueExtractorForBoolean {
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             int size = block.getValueCount(position);
             writeCount(values, size);
             if (size == 1 && inKey) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForBytesRef.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForBytesRef.java
index 97b2ce6da5e9b..65c5da5737a59 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForBytesRef.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForBytesRef.java
@@ -8,9 +8,9 @@
 package org.elasticsearch.compute.operator.topn;
 
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class ValueExtractorForBytesRef implements ValueExtractor {
     static ValueExtractorForBytesRef extractorFor(TopNEncoder encoder, boolean inKey, BytesRefBlock block) {
@@ -32,11 +32,11 @@ static ValueExtractorForBytesRef extractorFor(TopNEncoder encoder, boolean inKey
         this.inKey = inKey;
     }
 
-    protected final void writeCount(BytesRefBuilder values, int count) {
+    protected final void writeCount(BreakingBytesRefBuilder values, int count) {
         TopNEncoder.DEFAULT_UNSORTABLE.encodeVInt(count, values);
     }
 
-    protected final void actualWriteValue(BytesRefBuilder values, BytesRef value) {
+    protected final void actualWriteValue(BreakingBytesRefBuilder values, BytesRef value) {
         encoder.encodeBytesRef(value, values);
     }
 
@@ -49,7 +49,7 @@ static class ForVector extends ValueExtractorForBytesRef {
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             writeCount(values, 1);
             if (inKey) {
                 // will read results from the key
@@ -68,7 +68,7 @@ static class ForBlock extends ValueExtractorForBytesRef {
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             int size = block.getValueCount(position);
             writeCount(values, size);
             if (size == 1 && inKey) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForDouble.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForDouble.java
index 0bceeea462283..d20f2bf53972a 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForDouble.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForDouble.java
@@ -7,9 +7,9 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.DoubleVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class ValueExtractorForDouble implements ValueExtractor {
     static ValueExtractorForDouble extractorFor(TopNEncoder encoder, boolean inKey, DoubleBlock block) {
@@ -27,11 +27,11 @@ static ValueExtractorForDouble extractorFor(TopNEncoder encoder, boolean inKey,
         this.inKey = inKey;
     }
 
-    protected final void writeCount(BytesRefBuilder values, int count) {
+    protected final void writeCount(BreakingBytesRefBuilder values, int count) {
         TopNEncoder.DEFAULT_UNSORTABLE.encodeVInt(count, values);
     }
 
-    protected final void actualWriteValue(BytesRefBuilder values, double value) {
+    protected final void actualWriteValue(BreakingBytesRefBuilder values, double value) {
         TopNEncoder.DEFAULT_UNSORTABLE.encodeDouble(value, values);
     }
 
@@ -44,7 +44,7 @@ static class ForVector extends ValueExtractorForDouble {
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             writeCount(values, 1);
             if (inKey) {
                 // will read results from the key
@@ -63,7 +63,7 @@ static class ForBlock extends ValueExtractorForDouble {
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             int size = block.getValueCount(position);
             writeCount(values, size);
             if (size == 1 && inKey) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForInt.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForInt.java
index 28156ccb87cf7..d20368f874e8e 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForInt.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForInt.java
@@ -7,9 +7,9 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class ValueExtractorForInt implements ValueExtractor {
     static ValueExtractorForInt extractorFor(TopNEncoder encoder, boolean inKey, IntBlock block) {
@@ -27,11 +27,11 @@ static ValueExtractorForInt extractorFor(TopNEncoder encoder, boolean inKey, Int
         this.inKey = inKey;
     }
 
-    protected final void writeCount(BytesRefBuilder values, int count) {
+    protected final void writeCount(BreakingBytesRefBuilder values, int count) {
         TopNEncoder.DEFAULT_UNSORTABLE.encodeVInt(count, values);
     }
 
-    protected final void actualWriteValue(BytesRefBuilder values, int value) {
+    protected final void actualWriteValue(BreakingBytesRefBuilder values, int value) {
         TopNEncoder.DEFAULT_UNSORTABLE.encodeInt(value, values);
     }
 
@@ -44,7 +44,7 @@ static class ForVector extends ValueExtractorForInt {
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             writeCount(values, 1);
             if (inKey) {
                 // will read results from the key
@@ -63,7 +63,7 @@ static class ForBlock extends ValueExtractorForInt {
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             int size = block.getValueCount(position);
             writeCount(values, size);
             if (size == 1 && inKey) {
diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForLong.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForLong.java
index aec9aaf11c919..b7b566b3eda3d 100644
--- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForLong.java
+++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForLong.java
@@ -7,9 +7,9 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class ValueExtractorForLong implements ValueExtractor {
     static ValueExtractorForLong extractorFor(TopNEncoder encoder, boolean inKey, LongBlock block) {
@@ -27,11 +27,11 @@ static ValueExtractorForLong extractorFor(TopNEncoder encoder, boolean inKey, Lo
         this.inKey = inKey;
     }
 
-    protected final void writeCount(BytesRefBuilder values, int count) {
+    protected final void writeCount(BreakingBytesRefBuilder values, int count) {
         TopNEncoder.DEFAULT_UNSORTABLE.encodeVInt(count, values);
     }
 
-    protected final void actualWriteValue(BytesRefBuilder values, long value) {
+    protected final void actualWriteValue(BreakingBytesRefBuilder values, long value) {
         TopNEncoder.DEFAULT_UNSORTABLE.encodeLong(value, values);
     }
 
@@ -44,7 +44,7 @@ static class ForVector extends ValueExtractorForLong {
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             writeCount(values, 1);
             if (inKey) {
                 // will read results from the key
@@ -63,7 +63,7 @@ static class ForBlock extends ValueExtractorForLong {
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             int size = block.getValueCount(position);
             writeCount(values, size);
             if (size == 1 && inKey) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java
new file mode 100644
index 0000000000000..f13c3f9e444aa
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.core.Releasable;
+
+/**
+ * Builder for bytes arrays that checks its size against a {@link CircuitBreaker}.
+ */
+public class BreakingBytesRefBuilder implements Accountable, Releasable {
+    static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(BreakingBytesRefBuilder.class) + RamUsageEstimator
+        .shallowSizeOfInstance(BytesRef.class);
+
+    private final BytesRef bytes;
+    private final CircuitBreaker breaker;
+    private final String label;
+
+    /**
+     * Build.
+     * @param breaker the {@link CircuitBreaker} to check on resize
+     * @param label the label reported by the breaker when it breaks
+     */
+    public BreakingBytesRefBuilder(CircuitBreaker breaker, String label) {
+        /*
+         * We initialize BytesRef to a shared empty bytes array as is tradition.
+         * It's a good tradition. We don't know how big the thing will ultimately
+         * get, so we may as well let it grow.
+         *
+         * But! The ramBytesUsed mechanism doesn't know that this empty bytes is
+         * shared. So it overcounts. Which is *fine*. Empty bytes refs usually don't
+         * last long so it isn't worth making the accounting more complex to get it
+         * perfect. And overcounting in general isn't too bad compared to undercounting.
+         */
+        breaker.addEstimateBytesAndMaybeBreak(SHALLOW_SIZE + RamUsageEstimator.sizeOf(BytesRef.EMPTY_BYTES), label);
+        this.bytes = new BytesRef();
+        this.breaker = breaker;
+        this.label = label;
+    }
+
+    /**
+     * Make sure that the underlying bytes has a capacity of at least
+     * {@code capacity}.
+     */
+    public void grow(int capacity) {
+        int oldLength = bytes.bytes.length;
+        if (oldLength > capacity) {
+            return;
+        }
+        int newLength = ArrayUtil.oversize(capacity, Byte.BYTES);
+        breaker.addEstimateBytesAndMaybeBreak(
+            RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + newLength),
+            label
+        );
+        bytes.bytes = ArrayUtil.growExact(bytes.bytes, newLength);
+        breaker.addWithoutBreaking(-RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + oldLength));
+    }
+
+    /**
+     * Return the underlying bytes being built for direct manipulation.
+     * Callers will typically use this in combination with {@link #grow},
+     * {@link #length} and {@link #setLength} like this:
+     * {@code<pre>
+     *     bytesRefBuilder.grow(bytesRefBuilder.length() + Long.BYTES);
+     *     LONG.set(bytesRefBuilder.bytes(), bytesRefBuilder.length(), value);
+     *     bytesRefBuilder.setLength(bytesRefBuilder.length() + Long.BYTES);
+     * </pre>}
+     */
+    public byte[] bytes() {
+        return bytes.bytes;
+    }
+
+    /**
+     * The number of bytes in to this buffer. It is <strong>not></strong>
+     * the capacity of the buffer.
+     */
+    public int length() {
+        return bytes.length;
+    }
+
+    /**
+     * Set the number of bytes in this buffer. Does not deallocate the
+     * underlying buffer, only the length once built.
+     */
+    public void setLength(int length) {
+        bytes.length = length;
+    }
+
+    /**
+     * Append a byte.
+     */
+    public void append(byte b) {
+        grow(bytes.length + 1);
+        bytes.bytes[bytes.length++] = b;
+    }
+
+    /**
+     * Append bytes.
+     */
+    public void append(byte[] b, int off, int len) {
+        grow(bytes.length + len);
+        System.arraycopy(b, off, bytes.bytes, bytes.length, len);
+        bytes.length += len;
+    }
+
+    /**
+     * Append bytes.
+     */
+    public void append(BytesRef bytes) {
+        append(bytes.bytes, bytes.offset, bytes.length);
+    }
+
+    /**
+     * Reset the builder to an empty bytes array. Doesn't deallocate any memory.
+     */
+    public void clear() {
+        bytes.length = 0;
+    }
+
+    /**
+     * Returns a view of the data added as a {@link BytesRef}. Importantly, this does not
+     * copy the bytes and any further modification to the {@link BreakingBytesRefBuilder}
+     * will modify the returned {@link BytesRef}. The called must copy the bytes
+     * if they wish to keep them.
+     */
+    public BytesRef bytesRefView() {
+        assert bytes.offset == 0;
+        return bytes;
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        return SHALLOW_SIZE + RamUsageEstimator.sizeOf(bytes.bytes);
+    }
+
+    @Override
+    public void close() {
+        breaker.addWithoutBreaking(-ramBytesUsed());
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java
index 3b3ba69407065..8634d87e2932f 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java
@@ -8,11 +8,11 @@
 package org.elasticsearch.compute.operator.topn;
 
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 class DefaultSortableTopNEncoder extends SortableTopNEncoder {
     @Override
-    public int encodeBytesRef(BytesRef value, BytesRefBuilder bytesRefBuilder) {
+    public int encodeBytesRef(BytesRef value, BreakingBytesRefBuilder bytesRefBuilder) {
         throw new IllegalStateException("Cannot find encoder for BytesRef value");
     }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoder.java
index 668353b86519a..c23d5ae724999 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoder.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoder.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.compute.operator.topn;
 
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.VarHandle;
@@ -24,7 +24,7 @@ final class DefaultUnsortableTopNEncoder implements TopNEncoder {
     public static final VarHandle DOUBLE = MethodHandles.byteArrayViewVarHandle(double[].class, ByteOrder.nativeOrder());
 
     @Override
-    public void encodeLong(long value, BytesRefBuilder bytesRefBuilder) {
+    public void encodeLong(long value, BreakingBytesRefBuilder bytesRefBuilder) {
         bytesRefBuilder.grow(bytesRefBuilder.length() + Long.BYTES);
         LONG.set(bytesRefBuilder.bytes(), bytesRefBuilder.length(), value);
         bytesRefBuilder.setLength(bytesRefBuilder.length() + Long.BYTES);
@@ -46,7 +46,7 @@ public long decodeLong(BytesRef bytes) {
      * five bytes. Smaller values take fewer bytes. Negative numbers
      * will always use all 5 bytes.
      */
-    public void encodeVInt(int value, BytesRefBuilder bytesRefBuilder) {
+    public void encodeVInt(int value, BreakingBytesRefBuilder bytesRefBuilder) {
         while ((value & ~0x7F) != 0) {
             bytesRefBuilder.append(((byte) ((value & 0x7f) | 0x80)));
             value >>>= 7;
@@ -103,7 +103,7 @@ public int decodeVInt(BytesRef bytes) {
     }
 
     @Override
-    public void encodeInt(int value, BytesRefBuilder bytesRefBuilder) {
+    public void encodeInt(int value, BreakingBytesRefBuilder bytesRefBuilder) {
         bytesRefBuilder.grow(bytesRefBuilder.length() + Integer.BYTES);
         INT.set(bytesRefBuilder.bytes(), bytesRefBuilder.length(), value);
         bytesRefBuilder.setLength(bytesRefBuilder.length() + Integer.BYTES);
@@ -121,7 +121,7 @@ public int decodeInt(BytesRef bytes) {
     }
 
     @Override
-    public void encodeDouble(double value, BytesRefBuilder bytesRefBuilder) {
+    public void encodeDouble(double value, BreakingBytesRefBuilder bytesRefBuilder) {
         bytesRefBuilder.grow(bytesRefBuilder.length() + Double.BYTES);
         DOUBLE.set(bytesRefBuilder.bytes(), bytesRefBuilder.length(), value);
         bytesRefBuilder.setLength(bytesRefBuilder.length() + Long.BYTES);
@@ -139,7 +139,7 @@ public double decodeDouble(BytesRef bytes) {
     }
 
     @Override
-    public void encodeBoolean(boolean value, BytesRefBuilder bytesRefBuilder) {
+    public void encodeBoolean(boolean value, BreakingBytesRefBuilder bytesRefBuilder) {
         bytesRefBuilder.append(value ? (byte) 1 : (byte) 0);
     }
 
@@ -155,7 +155,7 @@ public boolean decodeBoolean(BytesRef bytes) {
     }
 
     @Override
-    public int encodeBytesRef(BytesRef value, BytesRefBuilder bytesRefBuilder) {
+    public int encodeBytesRef(BytesRef value, BreakingBytesRefBuilder bytesRefBuilder) {
         throw new UnsupportedOperationException();
     }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/FixedLengthTopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/FixedLengthTopNEncoder.java
index fbca1080c871c..943e32a484533 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/FixedLengthTopNEncoder.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/FixedLengthTopNEncoder.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.compute.operator.topn;
 
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 class FixedLengthTopNEncoder extends SortableTopNEncoder {
     private final int length;
@@ -18,7 +18,7 @@ class FixedLengthTopNEncoder extends SortableTopNEncoder {
     }
 
     @Override
-    public int encodeBytesRef(BytesRef value, BytesRefBuilder bytesRefBuilder) {
+    public int encodeBytesRef(BytesRef value, BreakingBytesRefBuilder bytesRefBuilder) {
         if (value.length != length) {
             throw new IllegalArgumentException("expected exactly [" + length + "] bytes but got [" + value.length + "]");
         }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractor.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractor.java
index 19daa1aba8d03..0d7d4d476d7b6 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractor.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractor.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
@@ -15,12 +14,13 @@
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 /**
- * Extracts keys into a {@link BytesRefBuilder}.
+ * Extracts keys into a {@link BreakingBytesRefBuilder}.
  */
 interface KeyExtractor {
-    int writeKey(BytesRefBuilder key, int position);
+    int writeKey(BreakingBytesRefBuilder key, int position);
 
     static KeyExtractor extractorFor(ElementType elementType, TopNEncoder encoder, boolean ascending, byte nul, byte nonNul, Block block) {
         if (false == (elementType == block.elementType() || ElementType.NULL == block.elementType())) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractorForNull.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractorForNull.java
index cea218f6036de..acf66942fe0f3 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractorForNull.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractorForNull.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 class KeyExtractorForNull implements KeyExtractor {
     private final byte nul;
@@ -17,7 +17,7 @@ class KeyExtractorForNull implements KeyExtractor {
     }
 
     @Override
-    public int writeKey(BytesRefBuilder values, int position) {
+    public int writeKey(BreakingBytesRefBuilder values, int position) {
         values.append(nul);
         return 1;
     }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/SortableTopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/SortableTopNEncoder.java
index 21b8b9d3d2e36..d04064e0a6777 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/SortableTopNEncoder.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/SortableTopNEncoder.java
@@ -8,15 +8,15 @@
 package org.elasticsearch.compute.operator.topn;
 
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 /**
  * A {@link TopNEncoder} that encodes values to byte arrays that may be sorted directly.
  */
 public abstract class SortableTopNEncoder implements TopNEncoder {
     @Override
-    public final void encodeLong(long value, BytesRefBuilder bytesRefBuilder) {
+    public final void encodeLong(long value, BreakingBytesRefBuilder bytesRefBuilder) {
         bytesRefBuilder.grow(bytesRefBuilder.length() + Long.BYTES);
         NumericUtils.longToSortableBytes(value, bytesRefBuilder.bytes(), bytesRefBuilder.length());
         bytesRefBuilder.setLength(bytesRefBuilder.length() + Long.BYTES);
@@ -34,7 +34,7 @@ public final long decodeLong(BytesRef bytes) {
     }
 
     @Override
-    public final void encodeInt(int value, BytesRefBuilder bytesRefBuilder) {
+    public final void encodeInt(int value, BreakingBytesRefBuilder bytesRefBuilder) {
         bytesRefBuilder.grow(bytesRefBuilder.length() + Integer.BYTES);
         NumericUtils.intToSortableBytes(value, bytesRefBuilder.bytes(), bytesRefBuilder.length());
         bytesRefBuilder.setLength(bytesRefBuilder.length() + Integer.BYTES);
@@ -52,7 +52,7 @@ public final int decodeInt(BytesRef bytes) {
     }
 
     @Override
-    public final void encodeDouble(double value, BytesRefBuilder bytesRefBuilder) {
+    public final void encodeDouble(double value, BreakingBytesRefBuilder bytesRefBuilder) {
         bytesRefBuilder.grow(bytesRefBuilder.length() + Long.BYTES);
         NumericUtils.longToSortableBytes(NumericUtils.doubleToSortableLong(value), bytesRefBuilder.bytes(), bytesRefBuilder.length());
         bytesRefBuilder.setLength(bytesRefBuilder.length() + Long.BYTES);
@@ -70,7 +70,7 @@ public final double decodeDouble(BytesRef bytes) {
     }
 
     @Override
-    public final void encodeBoolean(boolean value, BytesRefBuilder bytesRefBuilder) {
+    public final void encodeBoolean(boolean value, BreakingBytesRefBuilder bytesRefBuilder) {
         bytesRefBuilder.append(value ? (byte) 1 : (byte) 0);
     }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java
index 220a31a8fdac2..2d8f2666ff2f2 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java
@@ -9,7 +9,7 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 /**
  * Encodes values for {@link TopNOperator}. Some encoders encode values so sorting
@@ -41,23 +41,23 @@ public interface TopNEncoder {
      */
     VersionTopNEncoder VERSION = new VersionTopNEncoder();
 
-    void encodeLong(long value, BytesRefBuilder bytesRefBuilder);
+    void encodeLong(long value, BreakingBytesRefBuilder bytesRefBuilder);
 
     long decodeLong(BytesRef bytes);
 
-    void encodeInt(int value, BytesRefBuilder bytesRefBuilder);
+    void encodeInt(int value, BreakingBytesRefBuilder bytesRefBuilder);
 
     int decodeInt(BytesRef bytes);
 
-    void encodeDouble(double value, BytesRefBuilder bytesRefBuilder);
+    void encodeDouble(double value, BreakingBytesRefBuilder bytesRefBuilder);
 
     double decodeDouble(BytesRef bytes);
 
-    void encodeBoolean(boolean value, BytesRefBuilder bytesRefBuilder);
+    void encodeBoolean(boolean value, BreakingBytesRefBuilder bytesRefBuilder);
 
     boolean decodeBoolean(BytesRef bytes);
 
-    int encodeBytesRef(BytesRef value, BytesRefBuilder bytesRefBuilder);
+    int encodeBytesRef(BytesRef value, BreakingBytesRefBuilder bytesRefBuilder);
 
     BytesRef decodeBytesRef(BytesRef bytes, BytesRef scratch);
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNOperator.java
index acc4e90de6339..a324aba48b539 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNOperator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNOperator.java
@@ -9,14 +9,17 @@
 
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.PriorityQueue;
 import org.apache.lucene.util.RamUsageEstimator;
+import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.Operator;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -45,12 +48,15 @@ public class TopNOperator implements Operator, Accountable {
      * It mirrors somehow the Block build in the sense that it keeps around an array of offsets and a count of values (to account for
      * multivalues) to reference each position in each block of the Page.
      */
-    static final class Row implements Accountable {
-        private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(Row.class) + 2 * (RamUsageEstimator
-            .shallowSizeOfInstance(BytesRefBuilder.class) + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class)) + RamUsageEstimator
-                .shallowSizeOfInstance(BitSet.class);
+    static final class Row implements Accountable, Releasable {
+        private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(Row.class) + RamUsageEstimator
+            .shallowSizeOfInstance(BitSet.class);
+
+        /**
+         * The sort key.
+         */
+        final BreakingBytesRefBuilder keys;
 
-        final BytesRefBuilder keys = new BytesRefBuilder(); // BytesRef used to sort rows between each other
         /**
          * A true/false value (bit set/unset) for each byte in the BytesRef above corresponding to an asc/desc ordering.
          * For ex, if a Long is represented as 8 bytes, each of these bytes will have the same value (set/unset) if the respective Long
@@ -58,22 +64,50 @@ static final class Row implements Accountable {
          */
         final BitSet orderByCompositeKeyAscending = new BitSet();
 
-        final BytesRefBuilder values = new BytesRefBuilder();
+        /**
+         * Values to reconstruct the row. Sort of. When we reconstruct the row we read
+         * from both the {@link #keys} and the {@link #values}. So this only contains
+         * what is required to reconstruct the row that isn't already stored in {@link #values}.
+         */
+        final BreakingBytesRefBuilder values;
+
+        Row(CircuitBreaker breaker) {
+            boolean success = false;
+            try {
+                keys = new BreakingBytesRefBuilder(breaker, "topn");
+                values = new BreakingBytesRefBuilder(breaker, "topn");
+                success = true;
+            } finally {
+                if (success == false) {
+                    close();
+                }
+            }
+        }
 
         @Override
         public long ramBytesUsed() {
-            return SHALLOW_SIZE + RamUsageEstimator.sizeOf(keys.bytes()) + orderByCompositeKeyAscending.size() / Byte.SIZE
-                + RamUsageEstimator.sizeOf(values.bytes());
+            return SHALLOW_SIZE + keys.ramBytesUsed() + orderByCompositeKeyAscending.size() / Byte.SIZE + values.ramBytesUsed();
+        }
+
+        private void clear() {
+            keys.clear();
+            orderByCompositeKeyAscending.clear();
+            values.clear();
+        }
+
+        @Override
+        public void close() {
+            Releasables.closeExpectNoException(keys, values);
         }
     }
 
     record KeyFactory(KeyExtractor extractor, boolean ascending) {}
 
-    static final class RowFactory {
+    static final class RowFiller {
         private final ValueExtractor[] valueExtractors;
         private final KeyFactory[] keyFactories;
 
-        RowFactory(List<ElementType> elementTypes, List<TopNEncoder> encoders, List<SortOrder> sortOrders, Page page) {
+        RowFiller(List<ElementType> elementTypes, List<TopNEncoder> encoders, List<SortOrder> sortOrders, Page page) {
             valueExtractors = new ValueExtractor[page.getBlockCount()];
             for (int b = 0; b < valueExtractors.length; b++) {
                 valueExtractors[b] = ValueExtractor.extractorFor(
@@ -98,21 +132,12 @@ static final class RowFactory {
             }
         }
 
-        Row row(int position, Row spare) {
-            Row result;
-            if (spare == null) {
-                result = new Row();
-            } else {
-                result = spare;
-                result.keys.clear();
-                result.orderByCompositeKeyAscending.clear();
-                result.values.clear();
-            }
-
-            writeKey(position, result);
-            writeValues(position, result.values);
-
-            return result;
+        /**
+         * Fill a {@link Row} for {@code position}.
+         */
+        void row(int position, Row destination) {
+            writeKey(position, destination);
+            writeValues(position, destination.values);
         }
 
         private void writeKey(int position, Row row) {
@@ -128,7 +153,7 @@ private void writeKey(int position, Row row) {
             }
         }
 
-        private void writeValues(int position, BytesRefBuilder values) {
+        private void writeValues(int position, BreakingBytesRefBuilder values) {
             for (ValueExtractor e : valueExtractors) {
                 e.writeValue(values, position);
             }
@@ -180,7 +205,14 @@ public record TopNOperatorFactory(
 
         @Override
         public TopNOperator get(DriverContext driverContext) {
-            return new TopNOperator(topCount, elementTypes, encoders, sortOrders, maxPageSize);
+            return new TopNOperator(
+                driverContext.bigArrays().breakerService().getBreaker(CircuitBreaker.REQUEST),
+                topCount,
+                elementTypes,
+                encoders,
+                sortOrders,
+                maxPageSize
+            );
         }
 
         @Override
@@ -197,6 +229,7 @@ public String describe() {
         }
     }
 
+    private final CircuitBreaker breaker;
     private final Queue inputQueue;
 
     private final int maxPageSize;
@@ -208,12 +241,14 @@ public String describe() {
     private Iterator<Page> output;
 
     public TopNOperator(
+        CircuitBreaker breaker,
         int topCount,
         List<ElementType> elementTypes,
         List<TopNEncoder> encoders,
         List<SortOrder> sortOrders,
         int maxPageSize
     ) {
+        this.breaker = breaker;
         this.maxPageSize = maxPageSize;
         this.elementTypes = elementTypes;
         this.encoders = encoders;
@@ -224,8 +259,8 @@ public TopNOperator(
     static int compareRows(Row r1, Row r2) {
         // This is similar to r1.key.compareTo(r2.key) but stopping somewhere in the middle so that
         // we check the byte that mismatched
-        BytesRef br1 = r1.keys.get();
-        BytesRef br2 = r2.keys.get();
+        BytesRef br1 = r1.keys.bytesRefView();
+        BytesRef br2 = r2.keys.bytesRefView();
         int mismatchedByteIndex = Arrays.mismatch(
             br1.bytes,
             br1.offset,
@@ -249,10 +284,7 @@ static int compareRows(Row r1, Row r2) {
             }
         } else {
             // compare the byte that mismatched accounting for that respective byte asc/desc ordering
-            int c = Byte.compareUnsigned(
-                r1.keys.bytes()[br1.offset + mismatchedByteIndex],
-                r2.keys.bytes()[br2.offset + mismatchedByteIndex]
-            );
+            int c = Byte.compareUnsigned(br1.bytes[br1.offset + mismatchedByteIndex], br2.bytes[br2.offset + mismatchedByteIndex]);
             return r1.orderByCompositeKeyAscending.get(mismatchedByteIndex) ? -c : c;
         }
     }
@@ -264,12 +296,33 @@ public boolean needsInput() {
 
     @Override
     public void addInput(Page page) {
-        RowFactory rowFactory = new RowFactory(elementTypes, encoders, sortOrders, page);
-
-        Row removed = null;
-        for (int i = 0; i < page.getPositionCount(); i++) {
-            Row x = rowFactory.row(i, removed);
-            removed = inputQueue.insertWithOverflow(x);
+        RowFiller rowFiller = new RowFiller(elementTypes, encoders, sortOrders, page);
+
+        /*
+         * Since row tracks memory we have to be careful to close any unused rows,
+         * including any rows that fail while constructing because they allocate
+         * too much memory. The simplest way to manage that is this try/finally
+         * block and the mutable row variable here.
+         *
+         * If you exit the try/finally block and row is non-null it's always unused
+         * and must be closed. That happens either because it's overflow from the
+         * inputQueue or because we hit an allocation failure while building it.
+         */
+        Row row = null;
+        try {
+            for (int i = 0; i < page.getPositionCount(); i++) {
+                if (row == null) {
+                    row = new Row(breaker);
+                } else {
+                    row.keys.clear();
+                    row.orderByCompositeKeyAscending.clear();
+                    row.values.clear();
+                }
+                rowFiller.row(i, row);
+                row = inputQueue.insertWithOverflow(row);
+            }
+        } finally {
+            Releasables.close(row);
         }
     }
 
@@ -285,62 +338,70 @@ private Iterator<Page> toPages() {
             return Collections.emptyIterator();
         }
         List<Row> list = new ArrayList<>(inputQueue.size());
-        while (inputQueue.size() > 0) {
-            list.add(inputQueue.pop());
-        }
-        Collections.reverse(list);
-
-        List<Page> result = new ArrayList<>();
-        ResultBuilder[] builders = null;
-        int p = 0;
-        int size = 0;
-        for (int i = 0; i < list.size(); i++) {
-            if (builders == null) {
-                size = Math.min(maxPageSize, list.size() - i);
-                builders = new ResultBuilder[elementTypes.size()];
-                for (int b = 0; b < builders.length; b++) {
-                    builders[b] = ResultBuilder.resultBuilderFor(
-                        elementTypes.get(b),
-                        encoders.get(b).toUnsortable(),
-                        channelInKey(sortOrders, b),
-                        size
-                    );
-                }
-                p = 0;
+        try {
+            while (inputQueue.size() > 0) {
+                list.add(inputQueue.pop());
             }
+            Collections.reverse(list);
+
+            List<Page> result = new ArrayList<>();
+            ResultBuilder[] builders = null;
+            int p = 0;
+            int size = 0;
+            for (int i = 0; i < list.size(); i++) {
+                if (builders == null) {
+                    size = Math.min(maxPageSize, list.size() - i);
+                    builders = new ResultBuilder[elementTypes.size()];
+                    for (int b = 0; b < builders.length; b++) {
+                        builders[b] = ResultBuilder.resultBuilderFor(
+                            elementTypes.get(b),
+                            encoders.get(b).toUnsortable(),
+                            channelInKey(sortOrders, b),
+                            size
+                        );
+                    }
+                    p = 0;
+                }
 
-            Row row = list.get(i);
-            BytesRef keys = row.keys.get();
-            for (SortOrder so : sortOrders) {
-                if (keys.bytes[keys.offset] == so.nul()) {
+                Row row = list.get(i);
+                BytesRef keys = row.keys.bytesRefView();
+                for (SortOrder so : sortOrders) {
+                    if (keys.bytes[keys.offset] == so.nul()) {
+                        keys.offset++;
+                        keys.length--;
+                        continue;
+                    }
                     keys.offset++;
                     keys.length--;
-                    continue;
+                    builders[so.channel].decodeKey(keys);
+                }
+                if (keys.length != 0) {
+                    throw new IllegalArgumentException("didn't read all keys");
                 }
-                keys.offset++;
-                keys.length--;
-                builders[so.channel].decodeKey(keys);
-            }
-            if (keys.length != 0) {
-                throw new IllegalArgumentException("didn't read all keys");
-            }
 
-            BytesRef values = row.values.get();
-            for (ResultBuilder builder : builders) {
-                builder.decodeValue(values);
-            }
-            if (values.length != 0) {
-                throw new IllegalArgumentException("didn't read all values");
-            }
+                BytesRef values = row.values.bytesRefView();
+                for (ResultBuilder builder : builders) {
+                    builder.decodeValue(values);
+                }
+                if (values.length != 0) {
+                    throw new IllegalArgumentException("didn't read all values");
+                }
+
+                list.set(i, null);
+                row.close();
+
+                p++;
+                if (p == size) {
+                    result.add(new Page(Arrays.stream(builders).map(ResultBuilder::build).toArray(Block[]::new)));
+                    builders = null;
+                }
 
-            p++;
-            if (p == size) {
-                result.add(new Page(Arrays.stream(builders).map(ResultBuilder::build).toArray(Block[]::new)));
-                builders = null;
             }
+            assert builders == null;
+            return result.iterator();
+        } finally {
+            Releasables.closeExpectNoException(() -> Releasables.close(list));
         }
-        assert builders == null;
-        return result.iterator();
     }
 
     private static boolean channelInKey(List<SortOrder> sortOrders, int channel) {
@@ -367,7 +428,11 @@ public Page getOutput() {
 
     @Override
     public void close() {
-
+        /*
+         * If everything went well we'll have drained inputQueue to this'll
+         * be a noop. But if inputQueue
+         */
+        Releasables.closeExpectNoException(() -> Releasables.close(inputQueue));
     }
 
     private static long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(TopNOperator.class) + RamUsageEstimator
@@ -406,6 +471,10 @@ public String toString() {
             + "]";
     }
 
+    CircuitBreaker breaker() {
+        return breaker;
+    }
+
     private static class Queue extends PriorityQueue<Row> implements Accountable {
         private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(Queue.class);
         private final int maxSize;
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/UTF8TopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/UTF8TopNEncoder.java
index 3692e3009dd45..b2f3934aa87cd 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/UTF8TopNEncoder.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/UTF8TopNEncoder.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.compute.operator.topn;
 
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 import java.util.Arrays;
 
@@ -28,7 +28,7 @@ final class UTF8TopNEncoder extends SortableTopNEncoder {
     static final byte TERMINATOR = 0x00;
 
     @Override
-    public int encodeBytesRef(BytesRef value, BytesRefBuilder bytesRefBuilder) {
+    public int encodeBytesRef(BytesRef value, BreakingBytesRefBuilder bytesRefBuilder) {
         // add one bit to every byte so that there are no "0" bytes in the provided bytes. The only "0" bytes are
         // those defined as separators
         int end = value.offset + value.length;
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractor.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractor.java
index 2369078643ae4..af870dd336a74 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractor.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractor.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
@@ -16,12 +15,13 @@
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 /**
- * Extracts values into a {@link BytesRefBuilder}.
+ * Extracts values into a {@link BreakingBytesRefBuilder}.
  */
 interface ValueExtractor {
-    void writeValue(BytesRefBuilder values, int position);
+    void writeValue(BreakingBytesRefBuilder values, int position);
 
     static ValueExtractor extractorFor(ElementType elementType, TopNEncoder encoder, boolean inKey, Block block) {
         if (false == (elementType == block.elementType() || ElementType.NULL == block.elementType())) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractorForDoc.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractorForDoc.java
index 733b9cd4ab708..b6fc30e221cd7 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractorForDoc.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractorForDoc.java
@@ -7,8 +7,8 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.DocVector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 class ValueExtractorForDoc implements ValueExtractor {
     private final DocVector vector;
@@ -19,7 +19,7 @@ class ValueExtractorForDoc implements ValueExtractor {
     }
 
     @Override
-    public void writeValue(BytesRefBuilder values, int position) {
+    public void writeValue(BreakingBytesRefBuilder values, int position) {
         TopNEncoder.DEFAULT_UNSORTABLE.encodeInt(vector.shards().getInt(position), values);
         TopNEncoder.DEFAULT_UNSORTABLE.encodeInt(vector.segments().getInt(position), values);
         TopNEncoder.DEFAULT_UNSORTABLE.encodeInt(vector.docs().getInt(position), values);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractorForNull.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractorForNull.java
index 967ede1b3f46d..58d5b24e277a2 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractorForNull.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractorForNull.java
@@ -7,11 +7,11 @@
 
 package org.elasticsearch.compute.operator.topn;
 
-import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 class ValueExtractorForNull implements ValueExtractor {
     @Override
-    public void writeValue(BytesRefBuilder values, int position) {
+    public void writeValue(BreakingBytesRefBuilder values, int position) {
         /*
          * Write 0 values which can be read by *any* result builder and will always
          * make a null value.
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/VersionTopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/VersionTopNEncoder.java
index b6ce97586c449..d462f918bba92 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/VersionTopNEncoder.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/VersionTopNEncoder.java
@@ -8,11 +8,11 @@
 package org.elasticsearch.compute.operator.topn;
 
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 class VersionTopNEncoder extends SortableTopNEncoder {
     @Override
-    public int encodeBytesRef(BytesRef value, BytesRefBuilder bytesRefBuilder) {
+    public int encodeBytesRef(BytesRef value, BreakingBytesRefBuilder bytesRefBuilder) {
         // TODO versions can contain nul so we need to delegate to the utf-8 encoder for the utf-8 parts of a version
         for (int i = value.offset; i < value.length; i++) {
             if (value.bytes[i] == UTF8TopNEncoder.TERMINATOR) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/X-KeyExtractor.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/X-KeyExtractor.java.st
index 28f452ccac1a7..9ec03270da093 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/X-KeyExtractor.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/X-KeyExtractor.java.st
@@ -10,10 +10,10 @@ package org.elasticsearch.compute.operator.topn;
 $if(BytesRef)$
 import org.apache.lucene.util.BytesRef;
 $endif$
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.$Type$Block;
 import org.elasticsearch.compute.data.$Type$Vector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class KeyExtractorFor$Type$ implements KeyExtractor {
     static KeyExtractorFor$Type$ extractorFor(TopNEncoder encoder, boolean ascending, byte nul, byte nonNul, $Type$Block block) {
@@ -48,7 +48,7 @@ $endif$
         this.nonNul = nonNul;
     }
 
-    protected final int nonNul(BytesRefBuilder key, $type$ value) {
+    protected final int nonNul(BreakingBytesRefBuilder key, $type$ value) {
         key.append(nonNul);
 $if(BytesRef)$
         return encoder.encodeBytesRef(value, key) + 1;
@@ -61,7 +61,7 @@ $else$
 $endif$
     }
 
-    protected final int nul(BytesRefBuilder key) {
+    protected final int nul(BreakingBytesRefBuilder key) {
         key.append(nul);
         return 1;
     }
@@ -75,7 +75,7 @@ $endif$
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
 $if(BytesRef)$
             return nonNul(key, vector.get$Type$(position, scratch));
 $else$
@@ -93,7 +93,7 @@ $endif$
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -114,7 +114,7 @@ $endif$
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             if (block.isNull(position)) {
                 return nul(key);
             }
@@ -139,7 +139,7 @@ $endif$
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
@@ -187,7 +187,7 @@ $endif$
         }
 
         @Override
-        public int writeKey(BytesRefBuilder key, int position) {
+        public int writeKey(BreakingBytesRefBuilder key, int position) {
             int size = block.getValueCount(position);
             if (size == 0) {
                 return nul(key);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/X-ValueExtractor.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/X-ValueExtractor.java.st
index 3a8792387b142..0e25e44834c17 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/X-ValueExtractor.java.st
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/X-ValueExtractor.java.st
@@ -10,9 +10,9 @@ package org.elasticsearch.compute.operator.topn;
 $if(BytesRef)$
 import org.apache.lucene.util.BytesRef;
 $endif$
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.compute.data.$Type$Block;
 import org.elasticsearch.compute.data.$Type$Vector;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 
 abstract class ValueExtractorFor$Type$ implements ValueExtractor {
     static ValueExtractorFor$Type$ extractorFor(TopNEncoder encoder, boolean inKey, $Type$Block block) {
@@ -40,11 +40,11 @@ $endif$
         this.inKey = inKey;
     }
 
-    protected final void writeCount(BytesRefBuilder values, int count) {
+    protected final void writeCount(BreakingBytesRefBuilder values, int count) {
         TopNEncoder.DEFAULT_UNSORTABLE.encodeVInt(count, values);
     }
 
-    protected final void actualWriteValue(BytesRefBuilder values, $type$ value) {
+    protected final void actualWriteValue(BreakingBytesRefBuilder values, $type$ value) {
 $if(BytesRef)$
         encoder.encodeBytesRef(value, values);
 $else$
@@ -61,7 +61,7 @@ $endif$
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             writeCount(values, 1);
             if (inKey) {
                 // will read results from the key
@@ -84,7 +84,7 @@ $endif$
         }
 
         @Override
-        public void writeValue(BytesRefBuilder values, int position) {
+        public void writeValue(BreakingBytesRefBuilder values, int position) {
             int size = block.getValueCount(position);
             writeCount(values, size);
             if (size == 1 && inKey) {
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java
index 5edaa8d8da340..7bfeb57e6999a 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AnyOperatorTestCase.java
@@ -7,7 +7,9 @@
 
 package org.elasticsearch.compute.operator;
 
+import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.PageCacheRecycler;
@@ -80,9 +82,13 @@ public final void testSimpleToString() {
 
     /**
      * A {@link BigArrays} that won't throw {@link CircuitBreakingException}.
+     * <p>
+     *     Rather than using the {@link NoneCircuitBreakerService} we use a
+     *     very large limit so tests can call {@link CircuitBreaker#getUsed()}.
+     * </p>
      */
     protected final BigArrays nonBreakingBigArrays() {
-        return new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, new NoneCircuitBreakerService()).withCircuitBreaking();
+        return new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofBytes(Integer.MAX_VALUE)).withCircuitBreaking();
     }
 
     /**
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java
new file mode 100644
index 0000000000000..21e74e5a22fcf
--- /dev/null
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.compute.operator;
+
+import org.apache.lucene.tests.util.RamUsageTester;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class BreakingBytesRefBuilderTests extends ESTestCase {
+    public void testBreakOnBuild() {
+        String label = randomAlphaOfLength(4);
+        CircuitBreaker breaker = new MockBigArrays.LimitedBreaker(CircuitBreaker.REQUEST, ByteSizeValue.ofBytes(0));
+        Exception e = expectThrows(CircuitBreakingException.class, () -> new BreakingBytesRefBuilder(breaker, label));
+        assertThat(e.getMessage(), equalTo("over test limit"));
+    }
+
+    public void testAddByte() {
+        testAgainstOracle(() -> new TestIteration() {
+            byte b = randomByte();
+
+            @Override
+            public int size() {
+                return 1;
+            }
+
+            @Override
+            public void applyToBuilder(BreakingBytesRefBuilder builder) {
+                builder.append(b);
+            }
+
+            @Override
+            public void applyToOracle(BytesRefBuilder oracle) {
+                oracle.append(b);
+            }
+        });
+    }
+
+    public void testAddBytesRef() {
+        testAgainstOracle(() -> new TestIteration() {
+            BytesRef ref = new BytesRef(randomAlphaOfLengthBetween(1, 100));
+
+            @Override
+            public int size() {
+                return ref.length;
+            }
+
+            @Override
+            public void applyToBuilder(BreakingBytesRefBuilder builder) {
+                builder.append(ref);
+            }
+
+            @Override
+            public void applyToOracle(BytesRefBuilder oracle) {
+                oracle.append(ref);
+            }
+        });
+    }
+
+    public void testGrow() {
+        testAgainstOracle(() -> new TestIteration() {
+            int length = between(1, 100);
+            byte b = randomByte();
+
+            @Override
+            public int size() {
+                return length;
+            }
+
+            @Override
+            public void applyToBuilder(BreakingBytesRefBuilder builder) {
+                builder.grow(builder.length() + length);
+                builder.bytes()[builder.length()] = b;
+                builder.setLength(builder.length() + length);
+            }
+
+            @Override
+            public void applyToOracle(BytesRefBuilder oracle) {
+                oracle.grow(oracle.length() + length);
+                oracle.bytes()[oracle.length()] = b;
+                oracle.setLength(oracle.length() + length);
+            }
+        });
+    }
+
+    interface TestIteration {
+        int size();
+
+        void applyToBuilder(BreakingBytesRefBuilder builder);
+
+        void applyToOracle(BytesRefBuilder oracle);
+    }
+
+    private void testAgainstOracle(Supplier<TestIteration> iterations) {
+        int limit = between(1_000, 10_000);
+        String label = randomAlphaOfLength(4);
+        CircuitBreaker breaker = new MockBigArrays.LimitedBreaker(CircuitBreaker.REQUEST, ByteSizeValue.ofBytes(limit));
+        assertThat(breaker.getUsed(), equalTo(0L));
+        try (BreakingBytesRefBuilder builder = new BreakingBytesRefBuilder(breaker, label)) {
+            assertThat(breaker.getUsed(), equalTo(builder.ramBytesUsed()));
+            BytesRefBuilder oracle = new BytesRefBuilder();
+
+            assertThat(builder.bytesRefView(), equalTo(oracle.get()));
+            while (true) {
+                TestIteration iteration = iterations.get();
+                boolean willResize = builder.length() + iteration.size() >= builder.bytes().length;
+                if (willResize) {
+                    long resizeMemoryUsage = BreakingBytesRefBuilder.SHALLOW_SIZE + ramForArray(builder.bytes().length);
+                    resizeMemoryUsage += ramForArray(ArrayUtil.oversize(builder.bytes().length + iteration.size(), Byte.BYTES));
+                    if (resizeMemoryUsage > limit) {
+                        Exception e = expectThrows(CircuitBreakingException.class, () -> iteration.applyToBuilder(builder));
+                        assertThat(e.getMessage(), equalTo("over test limit"));
+                        break;
+                    }
+                }
+                iteration.applyToBuilder(builder);
+                iteration.applyToOracle(oracle);
+                assertThat(builder.bytesRefView(), equalTo(oracle.get()));
+                assertThat(
+                    builder.ramBytesUsed(),
+                    // Label and breaker aren't counted in ramBytesUsed because they are usually shared with other instances.
+                    equalTo(RamUsageTester.ramUsed(builder) - RamUsageTester.ramUsed(label) - RamUsageTester.ramUsed(breaker))
+                );
+                assertThat(builder.ramBytesUsed(), equalTo(breaker.getUsed()));
+            }
+        }
+        assertThat(breaker.getUsed(), equalTo(0L));
+    }
+
+    private long ramForArray(int length) {
+        return RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + length);
+    }
+}
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java
index 8edd2c7c83f64..8355a5a444bf5 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -62,14 +63,14 @@ public abstract class OperatorTestCase extends AnyOperatorTestCase {
      * are more likely to discover accidental behavior for clumped inputs.
      */
     public final void testSimpleSmallInput() {
-        assertSimple(nonBreakingBigArrays(), between(10, 100));
+        assertSimple(driverContext(), between(10, 100));
     }
 
     /**
      * Test a larger input set against {@link #simple}.
      */
     public final void testSimpleLargeInput() {
-        assertSimple(nonBreakingBigArrays(), between(1_000, 10_000));
+        assertSimple(driverContext(), between(1_000, 10_000));
     }
 
     /**
@@ -79,8 +80,12 @@ public final void testSimpleLargeInput() {
      */
     public final void testSimpleCircuitBreaking() {
         BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, smallEnoughToCircuitBreak());
-        Exception e = expectThrows(CircuitBreakingException.class, () -> assertSimple(bigArrays, between(1_000, 10_000)));
+        Exception e = expectThrows(
+            CircuitBreakingException.class,
+            () -> assertSimple(new DriverContext(bigArrays), between(1_000, 10_000))
+        );
         assertThat(e.getMessage(), equalTo(MockBigArrays.ERROR_MESSAGE));
+        assertThat(bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L));
     }
 
     /**
@@ -93,7 +98,7 @@ public final void testSimpleWithCranky() {
         CrankyCircuitBreakerService breaker = new CrankyCircuitBreakerService();
         BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, breaker).withCircuitBreaking();
         try {
-            assertSimple(bigArrays, between(1_000, 10_000));
+            assertSimple(new DriverContext(bigArrays), between(1_000, 10_000));
             // Either we get lucky and cranky doesn't throw and the test completes or we don't and it throws
         } catch (CircuitBreakingException e) {
             assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE));
@@ -129,10 +134,12 @@ protected final List<Page> oneDriverPerPageList(Iterator<List<Page>> source, Sup
         return result;
     }
 
-    private void assertSimple(BigArrays bigArrays, int size) {
+    private void assertSimple(DriverContext context, int size) {
         List<Page> input = CannedSourceOperator.collectPages(simpleInput(size));
-        List<Page> results = drive(simple(bigArrays.withCircuitBreaking()).get(driverContext()), input.iterator());
+        BigArrays bigArrays = context.bigArrays().withCircuitBreaking();
+        List<Page> results = drive(simple(bigArrays).get(context), input.iterator());
         assertSimpleOutput(input, results);
+        assertThat(bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L));
     }
 
     protected final List<Page> drive(Operator operator, Iterator<Page> input) {
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoderTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoderTests.java
index 9f46399546a09..6345758204d9f 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoderTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoderTests.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.compute.operator.topn;
 
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -39,10 +39,10 @@ public void testVIntNegative() {
     }
 
     private void testVInt(int v, int expectedBytes) {
-        BytesRefBuilder builder = new BytesRefBuilder();
+        BreakingBytesRefBuilder builder = ExtractorTests.nonBreakingBytesRefBuilder();
         TopNEncoder.DEFAULT_UNSORTABLE.encodeVInt(v, builder);
         assertThat(builder.length(), equalTo(expectedBytes));
-        BytesRef bytes = builder.toBytesRef();
+        BytesRef bytes = builder.bytesRefView();
         assertThat(TopNEncoder.DEFAULT_UNSORTABLE.decodeVInt(bytes), equalTo(v));
         assertThat(bytes.length, equalTo(0));
     }
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/ExtractorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/ExtractorTests.java
index 5fbb44f1fac0b..d79fde19f5487 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/ExtractorTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/ExtractorTests.java
@@ -11,13 +11,15 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockTestUtils;
 import org.elasticsearch.compute.data.BlockUtils;
 import org.elasticsearch.compute.data.DocVector;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -129,15 +131,19 @@ public ExtractorTests(TestCase testCase) {
         this.testCase = testCase;
     }
 
+    static BreakingBytesRefBuilder nonBreakingBytesRefBuilder() {
+        return new BreakingBytesRefBuilder(new NoopCircuitBreaker(CircuitBreaker.REQUEST), "topn");
+    }
+
     public void testNotInKey() {
         Block value = testCase.value.get();
 
-        BytesRefBuilder valuesBuilder = new BytesRefBuilder();
+        BreakingBytesRefBuilder valuesBuilder = nonBreakingBytesRefBuilder();
         ValueExtractor.extractorFor(testCase.type, testCase.encoder.toUnsortable(), false, value).writeValue(valuesBuilder, 0);
         assertThat(valuesBuilder.length(), greaterThan(0));
 
         ResultBuilder result = ResultBuilder.resultBuilderFor(testCase.type, testCase.encoder.toUnsortable(), false, 1);
-        BytesRef values = valuesBuilder.get();
+        BytesRef values = valuesBuilder.bytesRefView();
         result.decodeValue(values);
         assertThat(values.length, equalTo(0));
 
@@ -148,17 +154,17 @@ public void testInKey() {
         assumeFalse("can't sort on _doc", testCase.type == ElementType.DOC);
         Block value = testCase.value.get();
 
-        BytesRefBuilder keysBuilder = new BytesRefBuilder();
+        BreakingBytesRefBuilder keysBuilder = nonBreakingBytesRefBuilder();
         KeyExtractor.extractorFor(testCase.type, testCase.encoder.toSortable(), randomBoolean(), randomByte(), randomByte(), value)
             .writeKey(keysBuilder, 0);
         assertThat(keysBuilder.length(), greaterThan(0));
 
-        BytesRefBuilder valuesBuilder = new BytesRefBuilder();
+        BreakingBytesRefBuilder valuesBuilder = nonBreakingBytesRefBuilder();
         ValueExtractor.extractorFor(testCase.type, testCase.encoder.toUnsortable(), true, value).writeValue(valuesBuilder, 0);
         assertThat(valuesBuilder.length(), greaterThan(0));
 
         ResultBuilder result = ResultBuilder.resultBuilderFor(testCase.type, testCase.encoder.toUnsortable(), true, 1);
-        BytesRef keys = keysBuilder.get();
+        BytesRef keys = keysBuilder.bytesRefView();
         if (testCase.type == ElementType.NULL) {
             assertThat(keys.length, equalTo(1));
         } else {
@@ -168,7 +174,7 @@ public void testInKey() {
             result.decodeKey(keys);
             assertThat(keys.length, equalTo(0));
         }
-        BytesRef values = valuesBuilder.get();
+        BytesRef values = valuesBuilder.bytesRefView();
         result.decodeValue(values);
         assertThat(values.length, equalTo(0));
 
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNEncoderTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNEncoderTests.java
index 5f1dbc887a1d7..75fad6fffb5de 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNEncoderTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNEncoderTests.java
@@ -11,7 +11,7 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.versionfield.Version;
 
@@ -38,37 +38,37 @@ public TopNEncoderTests(TopNEncoder encoder) {
     }
 
     public void testLong() {
-        BytesRefBuilder builder = new BytesRefBuilder();
+        BreakingBytesRefBuilder builder = ExtractorTests.nonBreakingBytesRefBuilder();
         long v = randomLong();
         encoder.encodeLong(v, builder);
-        BytesRef encoded = builder.toBytesRef();
+        BytesRef encoded = builder.bytesRefView();
         assertThat(encoder.decodeLong(encoded), equalTo(v));
         assertThat(encoded.length, equalTo(0));
     }
 
     public void testInt() {
-        BytesRefBuilder builder = new BytesRefBuilder();
+        BreakingBytesRefBuilder builder = ExtractorTests.nonBreakingBytesRefBuilder();
         int v = randomInt();
         encoder.encodeInt(v, builder);
-        BytesRef encoded = builder.toBytesRef();
+        BytesRef encoded = builder.bytesRefView();
         assertThat(encoder.decodeInt(encoded), equalTo(v));
         assertThat(encoded.length, equalTo(0));
     }
 
     public void testDouble() {
-        BytesRefBuilder builder = new BytesRefBuilder();
+        BreakingBytesRefBuilder builder = ExtractorTests.nonBreakingBytesRefBuilder();
         double v = randomDouble();
         encoder.encodeDouble(v, builder);
-        BytesRef encoded = builder.toBytesRef();
+        BytesRef encoded = builder.bytesRefView();
         assertThat(encoder.decodeDouble(encoded), equalTo(v));
         assertThat(encoded.length, equalTo(0));
     }
 
     public void testBoolean() {
-        BytesRefBuilder builder = new BytesRefBuilder();
+        BreakingBytesRefBuilder builder = ExtractorTests.nonBreakingBytesRefBuilder();
         boolean v = randomBoolean();
         encoder.encodeBoolean(v, builder);
-        BytesRef encoded = builder.toBytesRef();
+        BytesRef encoded = builder.bytesRefView();
         assertThat(encoder.decodeBoolean(encoded), equalTo(v));
         assertThat(encoded.length, equalTo(0));
     }
@@ -110,9 +110,9 @@ public void testIp() {
     }
 
     private void roundTripBytesRef(BytesRef v) {
-        BytesRefBuilder builder = new BytesRefBuilder();
+        BreakingBytesRefBuilder builder = ExtractorTests.nonBreakingBytesRefBuilder();
         int reportedSize = encoder.encodeBytesRef(v, builder);
-        BytesRef encoded = builder.toBytesRef();
+        BytesRef encoded = builder.bytesRefView();
         assertThat(encoded.length, equalTo(reportedSize));
         assertThat(encoder.decodeBytesRef(encoded, new BytesRef()), equalTo(v));
         assertThat(encoded.length, equalTo(0));
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java
index f7ead4912d1be..a3e060f5693d8 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java
@@ -10,9 +10,11 @@
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.tests.util.RamUsageTester;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
@@ -141,7 +143,7 @@ protected String expectedToStringOfSimple() {
 
     @Override
     protected SourceOperator simpleInput(int size) {
-        return new SequenceLongBlockSourceOperator(LongStream.range(0, size).map(l -> ESTestCase.randomLong()));
+        return new SequenceLongBlockSourceOperator(LongStream.range(0, size).map(l -> ESTestCase.randomLong()), between(1, size * 2));
     }
 
     @Override
@@ -169,8 +171,11 @@ protected void assertSimpleOutput(List<Page> input, List<Page> results) {
 
     @Override
     protected ByteSizeValue smallEnoughToCircuitBreak() {
-        assumeTrue("TopN doesn't break the circuit breaker for now", false);
-        return ByteSizeValue.ZERO;
+        /*
+         * 775 causes us to blow up while collecting values and 780 doesn't
+         * trip the breaker. So 775 is the max on this range.
+         */
+        return ByteSizeValue.ofBytes(between(1, 775));
     }
 
     public void testRamBytesUsed() {
@@ -184,13 +189,15 @@ public void testRamBytesUsed() {
             List.of(new TopNOperator.SortOrder(0, true, false)),
             pageSize
         ).get(driverContext());
-        long actualEmpty = RamUsageTester.ramUsed(op) - RamUsageTester.ramUsed(LONG) - RamUsageTester.ramUsed(DEFAULT_UNSORTABLE);
+        long actualEmpty = RamUsageTester.ramUsed(op) - RamUsageTester.ramUsed(LONG) - RamUsageTester.ramUsed(DEFAULT_UNSORTABLE)
+            - RamUsageTester.ramUsed(op.breaker());
         assertThat(op.ramBytesUsed(), both(greaterThan(actualEmpty - underCount)).and(lessThan(actualEmpty)));
         // But when we fill it then we're quite close
         for (Page p : CannedSourceOperator.collectPages(simpleInput(topCount))) {
             op.addInput(p);
         }
-        long actualFull = RamUsageTester.ramUsed(op) - RamUsageTester.ramUsed(BYTES_REF) - RamUsageTester.ramUsed(DEFAULT_UNSORTABLE);
+        long actualFull = RamUsageTester.ramUsed(op) - RamUsageTester.ramUsed(LONG) - RamUsageTester.ramUsed(DEFAULT_UNSORTABLE)
+            - RamUsageTester.ramUsed(op.breaker());
         assertThat(op.ramBytesUsed(), both(greaterThan(actualFull - underCount)).and(lessThan(actualFull)));
     }
 
@@ -369,13 +376,15 @@ private TopNOperator.Row row(
         Page page,
         int position
     ) {
-        TopNOperator.RowFactory rf = new TopNOperator.RowFactory(
+        TopNOperator.RowFiller rf = new TopNOperator.RowFiller(
             IntStream.range(0, page.getBlockCount()).mapToObj(i -> elementType).toList(),
             IntStream.range(0, page.getBlockCount()).mapToObj(i -> encoder).toList(),
             List.of(new TopNOperator.SortOrder(channel, asc, nullsFirst)),
             page
         );
-        return rf.row(position, null);
+        TopNOperator.Row row = new TopNOperator.Row(nonBreakingBigArrays().breakerService().getBreaker("request"));
+        rf.row(position, row);
+        return row;
     }
 
     public void testTopNTwoColumns() {
@@ -459,6 +468,7 @@ public void testCollectAllValues() {
                 new CannedSourceOperator(List.of(new Page(blocks.toArray(Block[]::new))).iterator()),
                 List.of(
                     new TopNOperator(
+                        nonBreakingBigArrays().breakerService().getBreaker("request"),
                         topCount,
                         elementTypes,
                         encoders,
@@ -544,6 +554,7 @@ public void testCollectAllValues_RandomMultiValues() {
                 new CannedSourceOperator(List.of(new Page(blocks.toArray(Block[]::new))).iterator()),
                 List.of(
                     new TopNOperator(
+                        nonBreakingBigArrays().breakerService().getBreaker("request"),
                         topCount,
                         elementTypes,
                         encoders,
@@ -575,7 +586,16 @@ private List<Tuple<Long, Long>> topNTwoColumns(
             Driver driver = new Driver(
                 driverContext,
                 new TupleBlockSourceOperator(inputValues, randomIntBetween(1, 1000)),
-                List.of(new TopNOperator(limit, elementTypes, encoder, sortOrders, randomPageSize())),
+                List.of(
+                    new TopNOperator(
+                        nonBreakingBigArrays().breakerService().getBreaker("request"),
+                        limit,
+                        elementTypes,
+                        encoder,
+                        sortOrders,
+                        randomPageSize()
+                    )
+                ),
                 new PageConsumerOperator(page -> {
                     LongBlock block1 = page.getBlock(0);
                     LongBlock block2 = page.getBlock(1);
@@ -833,7 +853,16 @@ private void assertSortingOnMV(
             Driver driver = new Driver(
                 driverContext(),
                 new CannedSourceOperator(List.of(page).iterator()),
-                List.of(new TopNOperator(topCount, List.of(blockType), List.of(encoder), List.of(sortOrders), randomPageSize())),
+                List.of(
+                    new TopNOperator(
+                        nonBreakingBigArrays().breakerService().getBreaker("request"),
+                        topCount,
+                        List.of(blockType),
+                        List.of(encoder),
+                        List.of(sortOrders),
+                        randomPageSize()
+                    )
+                ),
                 new PageConsumerOperator(p -> readInto(actualValues, p)),
                 () -> {}
             )
@@ -936,7 +965,14 @@ public void testRandomMultiValuesTopN() {
 
         List<List<List<Object>>> actualValues = new ArrayList<>();
         List<Page> results = this.drive(
-            new TopNOperator(topCount, elementTypes, encoders, uniqueOrders.stream().toList(), rows),
+            new TopNOperator(
+                nonBreakingBigArrays().breakerService().getBreaker("request"),
+                topCount,
+                elementTypes,
+                encoders,
+                uniqueOrders.stream().toList(),
+                rows
+            ),
             List.of(new Page(blocks.toArray(Block[]::new))).iterator()
         );
         for (Page p : results) {
@@ -969,6 +1005,7 @@ public void testIPSortingSingleValue() throws UnknownHostException {
                 new CannedSourceOperator(List.of(new Page(builder.build())).iterator()),
                 List.of(
                     new TopNOperator(
+                        nonBreakingBigArrays().breakerService().getBreaker("request"),
                         ips.size(),
                         List.of(BYTES_REF),
                         List.of(TopNEncoder.IP),
@@ -1092,6 +1129,7 @@ private void assertIPSortingOnMultiValues(
                 new CannedSourceOperator(List.of(new Page(builder.build())).iterator()),
                 List.of(
                     new TopNOperator(
+                        nonBreakingBigArrays().breakerService().getBreaker("request"),
                         ips.size(),
                         List.of(BYTES_REF),
                         List.of(TopNEncoder.IP),
@@ -1173,6 +1211,7 @@ public void testZeroByte() {
                 new CannedSourceOperator(List.of(new Page(blocks.toArray(Block[]::new))).iterator()),
                 List.of(
                     new TopNOperator(
+                        nonBreakingBigArrays().breakerService().getBreaker("request"),
                         2,
                         List.of(BYTES_REF, INT),
                         List.of(TopNEncoder.UTF8, DEFAULT_UNSORTABLE),
@@ -1199,6 +1238,23 @@ public void testZeroByte() {
         assertThat((Integer) actual.get(1).get(1), equalTo(100));
     }
 
+    public void testCloseWithoutCompleting() {
+        CircuitBreaker breaker = new MockBigArrays.LimitedBreaker(CircuitBreaker.REQUEST, ByteSizeValue.ofGb(1));
+        try (
+            TopNOperator op = new TopNOperator(
+                breaker,
+                2,
+                List.of(INT),
+                List.of(DEFAULT_UNSORTABLE),
+                List.of(new TopNOperator.SortOrder(0, randomBoolean(), randomBoolean())),
+                randomPageSize()
+            )
+        ) {
+            op.addInput(new Page(new IntArrayVector(new int[] { 1 }, 1).asBlock()));
+        }
+        assertThat(breaker.getUsed(), equalTo(0L));
+    }
+
     @SuppressWarnings({ "unchecked", "rawtypes" })
     private static void readAsRows(List<List<List<Object>>> values, Page page) {
         if (page.getBlockCount() == 0) {
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNRowTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNRowTests.java
index be65cda3cce70..472b9e50767b1 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNRowTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNRowTests.java
@@ -8,30 +8,45 @@
 package org.elasticsearch.compute.operator.topn;
 
 import org.apache.lucene.tests.util.RamUsageTester;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.equalTo;
 
 public class TopNRowTests extends ESTestCase {
+    private final CircuitBreaker breaker = new NoopCircuitBreaker(CircuitBreaker.REQUEST);
+
     public void testRamBytesUsedEmpty() {
-        TopNOperator.Row row = new TopNOperator.Row();
-        // We double count the shared empty array for empty rows. This overcounting is *fine*, but throws off the test.
-        assertThat(row.ramBytesUsed(), equalTo(RamUsageTester.ramUsed(row) + RamUsageTester.ramUsed(new byte[0])));
+        TopNOperator.Row row = new TopNOperator.Row(breaker);
+        assertThat(row.ramBytesUsed(), equalTo(expectedRamBytesUsed(row)));
     }
 
     public void testRamBytesUsedSmall() {
-        TopNOperator.Row row = new TopNOperator.Row();
+        TopNOperator.Row row = new TopNOperator.Row(new NoopCircuitBreaker(CircuitBreaker.REQUEST));
         row.keys.append(randomByte());
         row.values.append(randomByte());
-        assertThat(row.ramBytesUsed(), equalTo(RamUsageTester.ramUsed(row)));
+        assertThat(row.ramBytesUsed(), equalTo(expectedRamBytesUsed(row)));
     }
 
     public void testRamBytesUsedBig() {
-        TopNOperator.Row row = new TopNOperator.Row();
+        TopNOperator.Row row = new TopNOperator.Row(new NoopCircuitBreaker(CircuitBreaker.REQUEST));
         for (int i = 0; i < 10000; i++) {
             row.keys.append(randomByte());
             row.values.append(randomByte());
         }
-        assertThat(row.ramBytesUsed(), equalTo(RamUsageTester.ramUsed(row)));
+        assertThat(row.ramBytesUsed(), equalTo(expectedRamBytesUsed(row)));
+    }
+
+    private long expectedRamBytesUsed(TopNOperator.Row row) {
+        long expected = RamUsageTester.ramUsed(row);
+        if (row.values.bytes().length == 0) {
+            // We double count the shared empty array for empty rows. This overcounting is *fine*, but throws off the test.
+            expected += RamUsageTester.ramUsed(new byte[0]);
+        }
+        // The breaker is shared infrastructure so we don't count it but RamUsageTester does
+        expected -= RamUsageTester.ramUsed(breaker);
+        expected -= RamUsageTester.ramUsed("topn");
+        return expected;
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
index 8a4a53f7b986d..8a18d7b3a26ed 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
@@ -12,9 +12,13 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.Driver;
@@ -104,6 +108,7 @@
 import static org.elasticsearch.xpack.esql.plugin.EsqlPlugin.ESQL_THREAD_POOL_NAME;
 import static org.elasticsearch.xpack.ql.CsvSpecReader.specParser;
 import static org.elasticsearch.xpack.ql.TestUtils.classpathResources;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -324,10 +329,11 @@ private ActualResults executePlan() throws Exception {
         String sessionId = "csv-test";
         ExchangeSourceHandler exchangeSource = new ExchangeSourceHandler(between(1, 64), threadPool.executor(ESQL_THREAD_POOL_NAME));
         ExchangeSinkHandler exchangeSink = new ExchangeSinkHandler(between(1, 64), threadPool::relativeTimeInMillis);
+        BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofGb(1)).withCircuitBreaking();
         LocalExecutionPlanner executionPlanner = new LocalExecutionPlanner(
             sessionId,
             new CancellableTask(1, "transport", "esql", null, TaskId.EMPTY_TASK_ID, Map.of()),
-            BigArrays.NON_RECYCLING_INSTANCE,
+            bigArrays,
             configuration,
             exchangeSource,
             exchangeSink,
@@ -391,6 +397,7 @@ protected void start(Driver driver, ActionListener<Void> driverListener) {
             return new ActualResults(columnNames, columnTypes, dataTypes, collectedPages, responseHeaders);
         } finally {
             Releasables.close(() -> Releasables.close(drivers), exchangeSource::decRef);
+            assertThat(bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L));
         }
     }
 

From 65db5dfabb1625bb8a698fd8433f459a1100a95a Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Mon, 18 Sep 2023 09:06:03 -0700
Subject: [PATCH 23/41] Bump versions after 8.10.1 release

---
 .buildkite/pipelines/intake.yml                  |  2 +-
 .buildkite/pipelines/periodic-packaging.yml      | 16 ++++++++++++++++
 .buildkite/pipelines/periodic.yml                | 10 ++++++++++
 .ci/bwcVersions                                  |  1 +
 .ci/snapshotBwcVersions                          |  2 +-
 .../src/main/java/org/elasticsearch/Version.java |  1 +
 6 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
index 92423539caf09..0f89ba9b8aeba 100644
--- a/.buildkite/pipelines/intake.yml
+++ b/.buildkite/pipelines/intake.yml
@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.14", "8.10.1", "8.11.0"]
+            BWC_VERSION: ["7.17.14", "8.10.2", "8.11.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
index 6730a2d60e7e9..1fd69ec8d8a2c 100644
--- a/.buildkite/pipelines/periodic-packaging.yml
+++ b/.buildkite/pipelines/periodic-packaging.yml
@@ -1616,6 +1616,22 @@ steps:
         env:
           BWC_VERSION: 8.10.1
 
+      - label: "{{matrix.image}} / 8.10.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.2
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.2
+
       - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
         timeout_in_minutes: 300
diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
index bc0dce06312a0..fe28b0a0cea64 100644
--- a/.buildkite/pipelines/periodic.yml
+++ b/.buildkite/pipelines/periodic.yml
@@ -992,6 +992,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.10.1
+      - label: 8.10.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.2#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.2
       - label: 8.11.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
         timeout_in_minutes: 300
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index 902c5356f9bae..ea42d7d4f0cc6 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -98,4 +98,5 @@ BWC_VERSION:
   - "8.9.2"
   - "8.10.0"
   - "8.10.1"
+  - "8.10.2"
   - "8.11.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
index bb57c7b896311..2620f98f1cec1 100644
--- a/.ci/snapshotBwcVersions
+++ b/.ci/snapshotBwcVersions
@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.14"
-  - "8.10.1"
+  - "8.10.2"
   - "8.11.0"
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index a1bb7df3fef0a..43b4d2c8bb37a 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -149,6 +149,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_9_2 = new Version(8_09_02_99);
     public static final Version V_8_10_0 = new Version(8_10_00_99);
     public static final Version V_8_10_1 = new Version(8_10_01_99);
+    public static final Version V_8_10_2 = new Version(8_10_02_99);
     public static final Version V_8_11_0 = new Version(8_11_00_99);
     public static final Version CURRENT = V_8_11_0;
 

From f3615978a74a0202ef0a6ab37bd916b88aeed9dd Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Mon, 18 Sep 2023 09:07:37 -0700
Subject: [PATCH 24/41] Prune changelogs after 8.10.1 release

---
 docs/changelog/99281.yaml | 5 -----
 docs/changelog/99480.yaml | 6 ------
 docs/changelog/99490.yaml | 6 ------
 docs/changelog/99491.yaml | 6 ------
 4 files changed, 23 deletions(-)
 delete mode 100644 docs/changelog/99281.yaml
 delete mode 100644 docs/changelog/99480.yaml
 delete mode 100644 docs/changelog/99490.yaml
 delete mode 100644 docs/changelog/99491.yaml

diff --git a/docs/changelog/99281.yaml b/docs/changelog/99281.yaml
deleted file mode 100644
index 12d018c63a863..0000000000000
--- a/docs/changelog/99281.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 99281
-summary: Fix PIT when resolving with deleted indices
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/99480.yaml b/docs/changelog/99480.yaml
deleted file mode 100644
index 08dcdceca60b0..0000000000000
--- a/docs/changelog/99480.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 99480
-summary: Fix deadlock between Cache.put and Cache.invalidateAll
-area: Infra/Core
-type: bug
-issues:
- - 99326
diff --git a/docs/changelog/99490.yaml b/docs/changelog/99490.yaml
deleted file mode 100644
index 07fd913f2c1c4..0000000000000
--- a/docs/changelog/99490.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 99490
-summary: Fork computation in `TransportGetShutdownStatusAction`
-area: Infra/Node Lifecycle
-type: bug
-issues:
- - 99487
diff --git a/docs/changelog/99491.yaml b/docs/changelog/99491.yaml
deleted file mode 100644
index dfeab5dbbad6d..0000000000000
--- a/docs/changelog/99491.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 99491
-summary: Use long in Centroid count
-area: Aggregations
-type: bug
-issues:
- - 80153

From d87b93f3174e82a86f4fad0ea8b85ac7b454f96d Mon Sep 17 00:00:00 2001
From: Joe Gallo <joe.gallo@elastic.co>
Date: Mon, 18 Sep 2023 12:22:56 -0400
Subject: [PATCH 25/41] Remove IngestDocument equals/hashCode (#99598)

---
 .../ingest/common/ForEachProcessorTests.java  |  4 +-
 .../ingest/common/GrokProcessorTests.java     |  2 +-
 .../ingest/WriteableIngestDocument.java       | 18 -----
 .../elasticsearch/ingest/IngestDocument.java  | 18 -----
 .../java/org/elasticsearch/script/CtxMap.java |  5 +-
 .../SimulateDocumentBaseResultTests.java      |  2 +-
 .../ingest/SimulateExecutionServiceTests.java |  2 +-
 .../ingest/SimulateProcessorResultTests.java  |  2 +-
 .../ingest/WriteableIngestDocumentTests.java  | 78 +++----------------
 .../ingest/IngestDocumentTests.java           | 77 ------------------
 .../ingest/PipelineProcessorTests.java        |  3 +-
 .../ingest/TrackingResultProcessorTests.java  | 22 +++---
 .../ingest/IngestDocumentMatcher.java         | 29 +++++--
 .../ingest/IngestDocumentMatcherTests.java    | 46 ++++++-----
 14 files changed, 82 insertions(+), 226 deletions(-)

diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
index 5f9936bd0c96d..fb580591ac8c7 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
@@ -344,7 +344,7 @@ public void testAppendingToTheSameField() {
         execProcessor(processor, ingestDocument, (result, e) -> {});
         assertThat(testProcessor.getInvokedCounter(), equalTo(2));
         ingestDocument.removeField("_ingest._value");
-        assertThat(ingestDocument, equalTo(originalIngestDocument));
+        assertIngestDocument(ingestDocument, originalIngestDocument);
     }
 
     public void testRemovingFromTheSameField() {
@@ -355,7 +355,7 @@ public void testRemovingFromTheSameField() {
         execProcessor(processor, ingestDocument, (result, e) -> {});
         assertThat(testProcessor.getInvokedCounter(), equalTo(2));
         ingestDocument.removeField("_ingest._value");
-        assertThat(ingestDocument, equalTo(originalIngestDocument));
+        assertIngestDocument(ingestDocument, originalIngestDocument);
     }
 
     public void testMapIteration() {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
index da71cb12a21f0..1e40345208a1b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
@@ -114,7 +114,7 @@ public void testMatchWithoutCaptures() throws Exception {
             MatcherWatchdog.noop()
         );
         processor.execute(doc);
-        assertThat(doc, equalTo(originalDoc));
+        assertIngestDocument(doc, originalDoc);
     }
 
     public void testNullField() {
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
index 40f8da6f7efaa..a77d0bf7e2b01 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
@@ -25,7 +25,6 @@
 import java.io.IOException;
 import java.time.ZonedDateTime;
 import java.util.Map;
-import java.util.Objects;
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
@@ -140,23 +139,6 @@ public static WriteableIngestDocument fromXContent(XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        WriteableIngestDocument that = (WriteableIngestDocument) o;
-        return Objects.equals(ingestDocument, that.ingestDocument);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(ingestDocument);
-    }
-
     @Override
     public String toString() {
         return ingestDocument.toString();
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
index 66931c7791bc8..1b2d8056ac437 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
@@ -911,24 +911,6 @@ public void doNoSelfReferencesCheck(boolean doNoSelfReferencesCheck) {
         this.doNoSelfReferencesCheck = doNoSelfReferencesCheck;
     }
 
-    @Override
-    public boolean equals(Object obj) {
-        if (obj == this) {
-            return true;
-        }
-        if (obj == null || getClass() != obj.getClass()) {
-            return false;
-        }
-
-        IngestDocument other = (IngestDocument) obj;
-        return Objects.equals(ctxMap, other.ctxMap) && Maps.deepEquals(ingestMetadata, other.ingestMetadata);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(ctxMap, ingestMetadata);
-    }
-
     @Override
     public String toString() {
         return "IngestDocument{" + " sourceAndMetadata=" + ctxMap + ", ingestMetadata=" + ingestMetadata + '}';
diff --git a/server/src/main/java/org/elasticsearch/script/CtxMap.java b/server/src/main/java/org/elasticsearch/script/CtxMap.java
index a379ba3db4faf..71c8ebea3fd36 100644
--- a/server/src/main/java/org/elasticsearch/script/CtxMap.java
+++ b/server/src/main/java/org/elasticsearch/script/CtxMap.java
@@ -8,7 +8,6 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
 
 import java.util.AbstractCollection;
@@ -337,9 +336,9 @@ public Object setValue(Object value) {
     public boolean equals(Object o) {
         if (this == o) return true;
         if ((o instanceof CtxMap) == false) return false;
+        if (super.equals(o) == false) return false;
         CtxMap<?> ctxMap = (CtxMap<?>) o;
-        if (Maps.deepEquals(this, ctxMap) == false) return false;
-        return Maps.deepEquals(source, ctxMap.source) && metadata.equals(ctxMap.metadata);
+        return source.equals(ctxMap.source) && metadata.equals(ctxMap.metadata);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java
index 4d06631e5065f..1c6a36d8d930d 100644
--- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java
@@ -88,7 +88,7 @@ protected Predicate<String> getRandomFieldsExcludeFilter() {
     }
 
     public static void assertEqualDocs(SimulateDocumentBaseResult response, SimulateDocumentBaseResult parsedResponse) {
-        assertEquals(response.getIngestDocument(), parsedResponse.getIngestDocument());
+        assertIngestDocument(response.getIngestDocument(), parsedResponse.getIngestDocument());
         if (response.getFailure() != null) {
             assertNotNull(parsedResponse.getFailure());
             assertThat(parsedResponse.getFailure().getMessage(), containsString(response.getFailure().getMessage()));
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
index 451ac526773da..f816d43c49240 100644
--- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
@@ -108,7 +108,7 @@ public void testExecuteItem() throws Exception {
         assertThat(processor.getInvokedCounter(), equalTo(2));
         assertThat(actualItemResponse, instanceOf(SimulateDocumentBaseResult.class));
         SimulateDocumentBaseResult simulateDocumentBaseResult = (SimulateDocumentBaseResult) actualItemResponse;
-        assertThat(simulateDocumentBaseResult.getIngestDocument(), equalTo(ingestDocument));
+        assertIngestDocument(simulateDocumentBaseResult.getIngestDocument(), ingestDocument);
         assertThat(simulateDocumentBaseResult.getFailure(), nullValue());
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
index d76df88e98189..e659893491d06 100644
--- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
@@ -127,7 +127,7 @@ protected Predicate<String> getRandomFieldsExcludeFilter() {
 
     static void assertEqualProcessorResults(SimulateProcessorResult response, SimulateProcessorResult parsedResponse) {
         assertEquals(response.getProcessorTag(), parsedResponse.getProcessorTag());
-        assertEquals(response.getIngestDocument(), parsedResponse.getIngestDocument());
+        assertIngestDocument(response.getIngestDocument(), parsedResponse.getIngestDocument());
         if (response.getFailure() != null) {
             assertNotNull(parsedResponse.getFailure());
             assertThat(parsedResponse.getFailure().getMessage(), containsString(response.getFailure().getMessage()));
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
index 593cfa92877ba..654ae9cdd68b7 100644
--- a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
@@ -26,7 +26,6 @@
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
@@ -45,72 +44,14 @@
 
 public class WriteableIngestDocumentTests extends AbstractXContentTestCase<WriteableIngestDocument> {
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99403")
-    public void testEqualsAndHashcode() throws Exception {
-        Map<String, Object> sourceAndMetadata = RandomDocumentPicks.randomSource(random());
-        int numFields = randomIntBetween(1, IngestDocument.Metadata.values().length);
-        sourceAndMetadata.put(VERSION.getFieldName(), TestIngestDocument.randomVersion());
-        for (int i = 0; i < numFields; i++) {
-            Tuple<String, Object> metadata = TestIngestDocument.randomMetadata();
-            sourceAndMetadata.put(metadata.v1(), metadata.v2());
-        }
-        Map<String, Object> ingestMetadata = new HashMap<>();
-        numFields = randomIntBetween(1, 5);
-        for (int i = 0; i < numFields; i++) {
-            ingestMetadata.put(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10));
-        }
-        WriteableIngestDocument ingestDocument = new WriteableIngestDocument(new IngestDocument(sourceAndMetadata, ingestMetadata));
-
-        boolean changed = false;
-        Map<String, Object> otherSourceAndMetadata;
-        if (randomBoolean()) {
-            otherSourceAndMetadata = RandomDocumentPicks.randomSource(random());
-            otherSourceAndMetadata.put(VERSION.getFieldName(), TestIngestDocument.randomVersion());
-            changed = true;
-        } else {
-            otherSourceAndMetadata = new HashMap<>(sourceAndMetadata);
-        }
-        if (randomBoolean()) {
-            numFields = randomIntBetween(1, IngestDocument.Metadata.values().length);
-            for (int i = 0; i < numFields; i++) {
-                Tuple<String, Object> metadata = randomValueOtherThanMany(
-                    t -> t.v2().equals(sourceAndMetadata.get(t.v1())),
-                    TestIngestDocument::randomMetadata
-                );
-                otherSourceAndMetadata.put(metadata.v1(), metadata.v2());
-            }
-            changed = true;
-        }
-
-        Map<String, Object> otherIngestMetadata;
-        if (randomBoolean()) {
-            otherIngestMetadata = new HashMap<>();
-            numFields = randomIntBetween(1, 5);
-            for (int i = 0; i < numFields; i++) {
-                otherIngestMetadata.put(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10));
-            }
-            changed = true;
-        } else {
-            otherIngestMetadata = Collections.unmodifiableMap(ingestMetadata);
-        }
+    @Override
+    protected boolean assertToXContentEquivalence() {
+        return false;
+    }
 
-        WriteableIngestDocument otherIngestDocument = new WriteableIngestDocument(
-            new IngestDocument(otherSourceAndMetadata, otherIngestMetadata)
-        );
-        if (changed) {
-            assertThat(ingestDocument, not(equalTo(otherIngestDocument)));
-            assertThat(otherIngestDocument, not(equalTo(ingestDocument)));
-        } else {
-            assertThat(ingestDocument, equalTo(otherIngestDocument));
-            assertThat(otherIngestDocument, equalTo(ingestDocument));
-            assertThat(ingestDocument.hashCode(), equalTo(otherIngestDocument.hashCode()));
-            WriteableIngestDocument thirdIngestDocument = new WriteableIngestDocument(
-                new IngestDocument(Collections.unmodifiableMap(sourceAndMetadata), Collections.unmodifiableMap(ingestMetadata))
-            );
-            assertThat(thirdIngestDocument, equalTo(ingestDocument));
-            assertThat(ingestDocument, equalTo(thirdIngestDocument));
-            assertThat(ingestDocument.hashCode(), equalTo(thirdIngestDocument.hashCode()));
-        }
+    @Override
+    protected void assertEqualInstances(WriteableIngestDocument expectedInstance, WriteableIngestDocument newInstance) {
+        assertIngestDocument(expectedInstance.getIngestDocument(), newInstance.getIngestDocument());
     }
 
     public void testSerialization() throws IOException {
@@ -166,8 +107,7 @@ public void testToXContent() throws IOException {
         sourceAndMetadata.putAll(toXContentSource);
         ingestDocument.getMetadata().keySet().forEach(k -> sourceAndMetadata.put(k, ingestDocument.getMetadata().get(k)));
         IngestDocument serializedIngestDocument = new IngestDocument(sourceAndMetadata, toXContentIngestMetadata);
-        // TODO(stu): is this test correct? Comparing against ingestDocument fails due to incorrectly failed byte array comparisons
-        assertThat(serializedIngestDocument, equalTo(serializedIngestDocument));
+        assertIngestDocument(writeableIngestDocument.getIngestDocument(), serializedIngestDocument);
     }
 
     public void testXContentHashSetSerialization() throws Exception {
@@ -189,7 +129,7 @@ public void testCopiesTheIngestDocument() {
         IngestDocument document = createRandomIngestDoc();
         WriteableIngestDocument wid = new WriteableIngestDocument(document);
 
-        assertThat(wid.getIngestDocument(), equalTo(document));
+        assertIngestDocument(wid.getIngestDocument(), document);
         assertThat(wid.getIngestDocument(), not(sameInstance(document)));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
index ae46668352048..166b830d14301 100644
--- a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
@@ -8,7 +8,6 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 import org.junit.Before;
@@ -958,70 +957,6 @@ public void testRemoveEmptyField() {
         }
     }
 
-    public void testEqualsAndHashcode() throws Exception {
-        Map<String, Object> sourceAndMetadata = RandomDocumentPicks.randomSource(random());
-        int numFields = randomIntBetween(1, IngestDocument.Metadata.values().length);
-        for (int i = 0; i < numFields; i++) {
-            Tuple<String, Object> metadata = TestIngestDocument.randomMetadata();
-            sourceAndMetadata.put(metadata.v1(), metadata.v2());
-        }
-        sourceAndMetadata.putIfAbsent("_version", TestIngestDocument.randomVersion());
-        Map<String, Object> ingestMetadata = new HashMap<>();
-        numFields = randomIntBetween(1, 5);
-        for (int i = 0; i < numFields; i++) {
-            ingestMetadata.put(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10));
-        }
-        // this is testing equality so use the wire constructor
-        IngestDocument ingestDocument = new IngestDocument(sourceAndMetadata, ingestMetadata);
-
-        boolean changed = false;
-        Map<String, Object> otherSourceAndMetadata;
-        if (randomBoolean()) {
-            otherSourceAndMetadata = RandomDocumentPicks.randomSource(random());
-            otherSourceAndMetadata.putIfAbsent("_version", TestIngestDocument.randomVersion());
-            changed = true;
-        } else {
-            otherSourceAndMetadata = new HashMap<>(sourceAndMetadata);
-        }
-        if (randomBoolean()) {
-            numFields = randomIntBetween(1, IngestDocument.Metadata.values().length);
-            for (int i = 0; i < numFields; i++) {
-                Tuple<String, Object> metadata;
-                do {
-                    metadata = TestIngestDocument.randomMetadata();
-                } while (metadata.v2().equals(sourceAndMetadata.get(metadata.v1()))); // must actually be a change
-                otherSourceAndMetadata.put(metadata.v1(), metadata.v2());
-            }
-            changed = true;
-        }
-
-        Map<String, Object> otherIngestMetadata;
-        if (randomBoolean()) {
-            otherIngestMetadata = new HashMap<>();
-            numFields = randomIntBetween(1, 5);
-            for (int i = 0; i < numFields; i++) {
-                otherIngestMetadata.put(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10));
-            }
-            changed = true;
-        } else {
-            otherIngestMetadata = Map.copyOf(ingestMetadata);
-        }
-
-        IngestDocument otherIngestDocument = new IngestDocument(otherSourceAndMetadata, otherIngestMetadata);
-        if (changed) {
-            assertThat(ingestDocument, not(equalTo(otherIngestDocument)));
-            assertThat(otherIngestDocument, not(equalTo(ingestDocument)));
-        } else {
-            assertThat(ingestDocument, equalTo(otherIngestDocument));
-            assertThat(otherIngestDocument, equalTo(ingestDocument));
-            assertThat(ingestDocument.hashCode(), equalTo(otherIngestDocument.hashCode()));
-            IngestDocument thirdIngestDocument = new IngestDocument(Map.copyOf(sourceAndMetadata), Map.copyOf(ingestMetadata));
-            assertThat(thirdIngestDocument, equalTo(ingestDocument));
-            assertThat(ingestDocument, equalTo(thirdIngestDocument));
-            assertThat(ingestDocument.hashCode(), equalTo(thirdIngestDocument.hashCode()));
-        }
-    }
-
     public void testIngestMetadataTimestamp() throws Exception {
         long before = System.currentTimeMillis();
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
@@ -1169,16 +1104,4 @@ public void testIndexHistory() {
         assertFalse(ingestDocument.updateIndexHistory(index1));
         assertThat(ingestDocument.getIndexHistory(), Matchers.contains(index1, index2));
     }
-
-    public void testEqualsAndHashCodeWithArray() {
-        // Test that equality still works when the ingest document uses primitive arrays,
-        // since normal .equals() methods would not work for Maps containing these arrays.
-        byte[] numbers = new byte[] { 0, 1, 2 };
-        ingestDocument.setFieldValue("some.nested.array", numbers);
-        IngestDocument copy = new IngestDocument(ingestDocument);
-        byte[] copiedNumbers = copy.getFieldValue("some.nested.array", byte[].class);
-        assertArrayEquals(numbers, copiedNumbers);
-        assertNotEquals(numbers, copiedNumbers);
-        assertThat(copy, equalTo(ingestDocument));
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java
index 40b1de8c09de4..ab7da6c952450 100644
--- a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java
@@ -20,6 +20,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.LongSupplier;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -60,7 +61,7 @@ public String getDescription() {
         Map<String, Object> config = new HashMap<>();
         config.put("name", pipelineId);
         factory.create(Map.of(), null, null, config).execute(testIngestDocument, (result, e) -> {});
-        assertEquals(testIngestDocument, invoked.get());
+        assertIngestDocument(testIngestDocument, invoked.get());
     }
 
     public void testThrowsOnMissingPipeline() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java
index 518543092db85..8ad8667a52770 100644
--- a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java
@@ -27,6 +27,7 @@
 import static org.elasticsearch.ingest.CompoundProcessor.ON_FAILURE_MESSAGE_FIELD;
 import static org.elasticsearch.ingest.CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD;
 import static org.elasticsearch.ingest.CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD;
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.elasticsearch.ingest.PipelineProcessorTests.createIngestService;
 import static org.elasticsearch.ingest.TrackingResultProcessor.decorate;
 import static org.hamcrest.Matchers.containsString;
@@ -66,7 +67,7 @@ public void testActualProcessor() throws Exception {
         assertThat(actualProcessor.getInvokedCounter(), equalTo(1));
         assertThat(resultList.size(), equalTo(1));
 
-        assertThat(resultList.get(0).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
+        assertIngestDocument(resultList.get(0).getIngestDocument(), expectedResult.getIngestDocument());
         assertThat(resultList.get(0).getFailure(), nullValue());
         assertThat(resultList.get(0).getProcessorTag(), equalTo(expectedResult.getProcessorTag()));
     }
@@ -222,7 +223,7 @@ public void testActualCompoundProcessorWithIgnoreFailure() throws Exception {
         );
         assertThat(testProcessor.getInvokedCounter(), equalTo(1));
         assertThat(resultList.size(), equalTo(1));
-        assertThat(resultList.get(0).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
+        assertIngestDocument(resultList.get(0).getIngestDocument(), expectedResult.getIngestDocument());
         assertThat(resultList.get(0).getFailure(), sameInstance(exception));
         assertThat(resultList.get(0).getProcessorTag(), equalTo(expectedResult.getProcessorTag()));
     }
@@ -281,7 +282,7 @@ public void testActualCompoundProcessorWithFalseConditional() throws Exception {
         assertFalse(resultList.get(2).getIngestDocument().hasField(key2));
         assertTrue(resultList.get(2).getIngestDocument().hasField(key3));
 
-        assertThat(resultList.get(2).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
+        assertIngestDocument(resultList.get(2).getIngestDocument(), expectedResult.getIngestDocument());
         assertThat(resultList.get(2).getFailure(), nullValue());
         assertThat(resultList.get(2).getProcessorTag(), nullValue());
     }
@@ -335,7 +336,7 @@ public void testActualPipelineProcessor() throws Exception {
         assertTrue(resultList.get(2).getIngestDocument().hasField(key2));
         assertFalse(resultList.get(2).getIngestDocument().hasField(key3));
 
-        assertThat(resultList.get(3).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
+        assertIngestDocument(resultList.get(3).getIngestDocument(), expectedResult.getIngestDocument());
         assertThat(resultList.get(3).getFailure(), nullValue());
         assertThat(resultList.get(3).getProcessorTag(), nullValue());
     }
@@ -425,7 +426,7 @@ public void testActualPipelineProcessorWithTrueConditional() throws Exception {
         assertTrue(resultList.get(3).getIngestDocument().hasField(key2));
         assertFalse(resultList.get(3).getIngestDocument().hasField(key3));
 
-        assertThat(resultList.get(4).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
+        assertIngestDocument(resultList.get(4).getIngestDocument(), expectedResult.getIngestDocument());
         assertThat(resultList.get(4).getFailure(), nullValue());
         assertThat(resultList.get(4).getProcessorTag(), nullValue());
     }
@@ -511,7 +512,7 @@ public void testActualPipelineProcessorWithFalseConditional() throws Exception {
         assertThat(resultList.get(2).getConditionalWithResult().v1(), equalTo(scriptName));
         assertThat(resultList.get(2).getConditionalWithResult().v2(), is(Boolean.FALSE));
 
-        assertThat(resultList.get(3).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
+        assertIngestDocument(resultList.get(3).getIngestDocument(), expectedResult.getIngestDocument());
         assertThat(resultList.get(3).getFailure(), nullValue());
         assertThat(resultList.get(3).getProcessorTag(), nullValue());
     }
@@ -577,7 +578,7 @@ public void testActualPipelineProcessorWithHandledFailure() throws Exception {
         assertTrue(resultList.get(3).getIngestDocument().hasField(key2));
         assertFalse(resultList.get(3).getIngestDocument().hasField(key3));
 
-        assertThat(resultList.get(4).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
+        assertIngestDocument(resultList.get(4).getIngestDocument(), expectedResult.getIngestDocument());
         assertThat(resultList.get(4).getFailure(), nullValue());
         assertThat(resultList.get(4).getProcessorTag(), nullValue());
     }
@@ -710,14 +711,17 @@ public void testActualPipelineProcessorRepeatedInvocation() throws Exception {
         assertNull(resultList.get(0).getConditionalWithResult());
         assertThat(resultList.get(0).getType(), equalTo("pipeline"));
 
-        assertThat(resultList.get(1).getIngestDocument(), not(equalTo(expectedResult.getIngestDocument())));
+        assertThat(
+            resultList.get(1).getIngestDocument().getFieldValue(key1, Integer.class),
+            not(equalTo(expectedResult.getIngestDocument().getFieldValue(key1, Integer.class)))
+        );
         assertThat(resultList.get(1).getFailure(), nullValue());
         assertThat(resultList.get(1).getProcessorTag(), nullValue());
 
         assertNull(resultList.get(2).getConditionalWithResult());
         assertThat(resultList.get(2).getType(), equalTo("pipeline"));
 
-        assertThat(resultList.get(3).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
+        assertIngestDocument(resultList.get(3).getIngestDocument(), expectedResult.getIngestDocument());
         assertThat(resultList.get(3).getFailure(), nullValue());
         assertThat(resultList.get(3).getProcessorTag(), nullValue());
 
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java b/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java
index e2bea702baff5..f4e88d6eb4fb5 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java
@@ -12,17 +12,32 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class IngestDocumentMatcher {
+public final class IngestDocumentMatcher {
+
+    private IngestDocumentMatcher() {
+        // utility class
+    }
+
     /**
      * Helper method to assert the equivalence between two IngestDocuments.
      *
-     * @param docA first document to compare
-     * @param docB second document to compare
+     * @param expected first document to compare
+     * @param actual second document to compare
      */
-    public static void assertIngestDocument(IngestDocument docA, IngestDocument docB) {
-        if ((deepEquals(docA.getIngestMetadata(), docB.getIngestMetadata(), true)
-            && deepEquals(docA.getSourceAndMetadata(), docB.getSourceAndMetadata(), false)) == false) {
-            throw new AssertionError("Expected [" + docA + "] but received [" + docB + "].");
+    public static void assertIngestDocument(IngestDocument expected, IngestDocument actual) {
+        // trivially true: if they're both null, then all is well
+        if (expected == null && actual == null) {
+            return;
+        }
+
+        // if only one is null, however, then that's not okay
+        if ((expected == null || actual == null)) {
+            throw new AssertionError("Expected [" + expected + "] but received [" + actual + "].");
+        }
+
+        if ((deepEquals(expected.getIngestMetadata(), actual.getIngestMetadata(), true)
+            && deepEquals(expected.getSourceAndMetadata(), actual.getSourceAndMetadata(), false)) == false) {
+            throw new AssertionError("Expected [" + expected + "] but received [" + actual + "].");
         }
     }
 
diff --git a/test/framework/src/test/java/org/elasticsearch/ingest/IngestDocumentMatcherTests.java b/test/framework/src/test/java/org/elasticsearch/ingest/IngestDocumentMatcherTests.java
index 019f64636831d..a01f86d8a6b28 100644
--- a/test/framework/src/test/java/org/elasticsearch/ingest/IngestDocumentMatcherTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/ingest/IngestDocumentMatcherTests.java
@@ -10,9 +10,8 @@
 
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
@@ -29,41 +28,52 @@ public void testDifferentMapData() {
 
     public void testDifferentLengthListData() {
         String rootKey = "foo";
-        IngestDocument document1 = TestIngestDocument.withDefaultVersion(Collections.singletonMap(rootKey, Arrays.asList("bar", "baz")));
-        IngestDocument document2 = TestIngestDocument.withDefaultVersion(Collections.singletonMap(rootKey, Collections.emptyList()));
+        IngestDocument document1 = TestIngestDocument.withDefaultVersion(Map.of(rootKey, List.of("bar", "baz")));
+        IngestDocument document2 = TestIngestDocument.withDefaultVersion(Map.of(rootKey, List.of()));
         assertThrowsOnComparision(document1, document2);
     }
 
     public void testDifferentNestedListFieldData() {
         String rootKey = "foo";
-        IngestDocument document1 = TestIngestDocument.withDefaultVersion(Collections.singletonMap(rootKey, Arrays.asList("bar", "baz")));
-        IngestDocument document2 = TestIngestDocument.withDefaultVersion(Collections.singletonMap(rootKey, Arrays.asList("bar", "blub")));
+        IngestDocument document1 = TestIngestDocument.withDefaultVersion(Map.of(rootKey, List.of("bar", "baz")));
+        IngestDocument document2 = TestIngestDocument.withDefaultVersion(Map.of(rootKey, List.of("bar", "blub")));
         assertThrowsOnComparision(document1, document2);
     }
 
     public void testDifferentNestedMapFieldData() {
         String rootKey = "foo";
-        IngestDocument document1 = TestIngestDocument.withDefaultVersion(
-            Collections.singletonMap(rootKey, Collections.singletonMap("bar", "baz"))
-        );
-        IngestDocument document2 = TestIngestDocument.withDefaultVersion(
-            Collections.singletonMap(rootKey, Collections.singletonMap("bar", "blub"))
-        );
+        IngestDocument document1 = TestIngestDocument.withDefaultVersion(Map.of(rootKey, Map.of("bar", "baz")));
+        IngestDocument document2 = TestIngestDocument.withDefaultVersion(Map.of(rootKey, Map.of("bar", "blub")));
         assertThrowsOnComparision(document1, document2);
     }
 
     public void testOnTypeConflict() {
         String rootKey = "foo";
-        IngestDocument document1 = TestIngestDocument.withDefaultVersion(
-            Collections.singletonMap(rootKey, Collections.singletonList("baz"))
-        );
-        IngestDocument document2 = TestIngestDocument.withDefaultVersion(
-            Collections.singletonMap(rootKey, Collections.singletonMap("blub", "blab"))
-        );
+        IngestDocument document1 = TestIngestDocument.withDefaultVersion(Map.of(rootKey, List.of("baz")));
+        IngestDocument document2 = TestIngestDocument.withDefaultVersion(Map.of(rootKey, Map.of("blub", "blab")));
         assertThrowsOnComparision(document1, document2);
     }
 
+    public void testNestedMapArrayEquivalence() {
+        IngestDocument ingestDocument = TestIngestDocument.emptyIngestDocument();
+        // Test that equality still works when the ingest document uses primitive arrays,
+        // since normal .equals() methods would not work for Maps containing these arrays.
+        byte[] numbers = new byte[] { 0, 1, 2 };
+        ingestDocument.setFieldValue("some.nested.array", numbers);
+        IngestDocument copy = new IngestDocument(ingestDocument);
+        byte[] copiedNumbers = copy.getFieldValue("some.nested.array", byte[].class);
+        assertArrayEquals(numbers, copiedNumbers);
+        assertNotEquals(numbers, copiedNumbers);
+        assertIngestDocument(ingestDocument, copy);
+    }
+
+    public void testNullsAreEqual() {
+        assertIngestDocument(null, null);
+    }
+
     private static void assertThrowsOnComparision(IngestDocument document1, IngestDocument document2) {
+        expectThrows(AssertionError.class, () -> assertIngestDocument(document1, null));
+        expectThrows(AssertionError.class, () -> assertIngestDocument(null, document2));
         expectThrows(AssertionError.class, () -> assertIngestDocument(document1, document2));
         expectThrows(AssertionError.class, () -> assertIngestDocument(document2, document1));
     }

From 88e862fa64d7ea4477685837b8c54e385588a007 Mon Sep 17 00:00:00 2001
From: James Rodewig <james.rodewig@elastic.co>
Date: Mon, 18 Sep 2023 13:47:47 -0400
Subject: [PATCH 26/41] [main] [DOCS] Add 8.10.1 release notes (#99634)
 (#99636)

---
 docs/reference/release-notes.asciidoc        |  2 ++
 docs/reference/release-notes/8.10.1.asciidoc | 22 ++++++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 docs/reference/release-notes/8.10.1.asciidoc

diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
index df8c167a1ae88..cb02320529457 100644
--- a/docs/reference/release-notes.asciidoc
+++ b/docs/reference/release-notes.asciidoc
@@ -7,6 +7,7 @@
 This section summarizes the changes in each release.
 
 * <<release-notes-8.11.0>>
+* <<release-notes-8.10.1>>
 * <<release-notes-8.10.0>>
 * <<release-notes-8.9.2>>
 * <<release-notes-8.9.1>>
@@ -50,6 +51,7 @@ This section summarizes the changes in each release.
 --
 
 include::release-notes/8.11.0.asciidoc[]
+include::release-notes/8.10.1.asciidoc[]
 include::release-notes/8.10.0.asciidoc[]
 include::release-notes/8.9.2.asciidoc[]
 include::release-notes/8.9.1.asciidoc[]
diff --git a/docs/reference/release-notes/8.10.1.asciidoc b/docs/reference/release-notes/8.10.1.asciidoc
new file mode 100644
index 0000000000000..de890ccadd772
--- /dev/null
+++ b/docs/reference/release-notes/8.10.1.asciidoc
@@ -0,0 +1,22 @@
+[[release-notes-8.10.1]]
+== {es} version 8.10.1
+
+Also see <<breaking-changes-8.10,Breaking changes in 8.10>>.
+
+[[bug-8.10.1]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Use long in Centroid count {es-pull}99491[#99491] (issue: {es-issue}80153[#80153])
+
+Infra/Core::
+* Fix deadlock between Cache.put and Cache.invalidateAll {es-pull}99480[#99480] (issue: {es-issue}99326[#99326])
+
+Infra/Node Lifecycle::
+* Fork computation in `TransportGetShutdownStatusAction` {es-pull}99490[#99490] (issue: {es-issue}99487[#99487])
+
+Search::
+* Fix PIT when resolving with deleted indices {es-pull}99281[#99281]
+
+

From f3aa254b55747c704d56dc8fb87ba8c31880d9f6 Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Mon, 18 Sep 2023 14:03:59 -0500
Subject: [PATCH 27/41] Bump version for commons-compress (#99633)

This commit bumps the version for commons-compress to 1.24.0.
---
 distribution/tools/geoip-cli/build.gradle | 2 +-
 gradle/build.versions.toml                | 2 +-
 gradle/verification-metadata.xml          | 6 +++---
 modules/ingest-attachment/build.gradle    | 2 +-
 test/x-content/build.gradle               | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/distribution/tools/geoip-cli/build.gradle b/distribution/tools/geoip-cli/build.gradle
index ebc34550a6625..cc7ac34a8acb0 100644
--- a/distribution/tools/geoip-cli/build.gradle
+++ b/distribution/tools/geoip-cli/build.gradle
@@ -17,5 +17,5 @@ dependencies {
   compileOnly project(":libs:elasticsearch-cli")
   compileOnly project(":libs:elasticsearch-x-content")
   testImplementation project(":test:framework")
-  testImplementation "org.apache.commons:commons-compress:1.21"
+  testImplementation "org.apache.commons:commons-compress:1.24.0"
 }
diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml
index 532e75c39d3c9..97cefe4df8f67 100644
--- a/gradle/build.versions.toml
+++ b/gradle/build.versions.toml
@@ -7,7 +7,7 @@ spock = "2.1-groovy-3.0"
 [libraries]
 ant = "org.apache.ant:ant:1.10.12"
 antlrst4 = "org.antlr:ST4:4.3.4"
-apache-compress = "org.apache.commons:commons-compress:1.21"
+apache-compress = "org.apache.commons:commons-compress:1.24.0"
 apache-rat = "org.apache.rat:apache-rat:0.11"
 asm = { group = "org.ow2.asm", name="asm", version.ref="asm" }
 asm-tree = { group = "org.ow2.asm", name="asm-tree", version.ref="asm" }
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index 7a894242f785b..6cd8d56be9530 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -1690,9 +1690,9 @@
             <sha256 value="6aecfd5459728a595601cfa07258d131972ffc39b492eb48bdd596577a2f244a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.commons" name="commons-compress" version="1.22">
-         <artifact name="commons-compress-1.22.jar">
-            <sha256 value="53d04a0efc7223baecaa303bd5d298eb0600e6b82b4076f9cecd558b97ba760b" origin="Generated by Gradle"/>
+      <component group="org.apache.commons" name="commons-compress" version="1.24.0">
+         <artifact name="commons-compress-1.24.0.jar">
+            <sha256 value="fbf2c5d275e513c08f4e4fb539726dbf3ac2142af956ba93efb983244f0c36bd" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.commons" name="commons-compress" version="1.4.1">
diff --git a/modules/ingest-attachment/build.gradle b/modules/ingest-attachment/build.gradle
index 09e0184a35716..461474fa68dca 100644
--- a/modules/ingest-attachment/build.gradle
+++ b/modules/ingest-attachment/build.gradle
@@ -76,7 +76,7 @@ dependencies {
   // MS Office
   api "org.apache.poi:poi-scratchpad:${versions.poi}"
   // Apple iWork
-  api 'org.apache.commons:commons-compress:1.22'
+  api 'org.apache.commons:commons-compress:1.24.0'
   // Outlook documents
   api "org.apache.james:apache-mime4j-core:${versions.mime4j}"
   api "org.apache.james:apache-mime4j-dom:${versions.mime4j}"
diff --git a/test/x-content/build.gradle b/test/x-content/build.gradle
index 5f0975cfdf1b2..9c00e32b41348 100644
--- a/test/x-content/build.gradle
+++ b/test/x-content/build.gradle
@@ -18,7 +18,7 @@ dependencies {
   implementation "com.networknt:json-schema-validator:${versions.networknt_json_schema_validator}"
   implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
   implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-  implementation "org.apache.commons:commons-compress:1.21"
+  implementation "org.apache.commons:commons-compress:1.24.0"
   implementation "org.apache.commons:commons-lang3:${versions.commons_lang3}"
 }
 

From 15614e60c0f7bd82554668eb0319df7e944acb1d Mon Sep 17 00:00:00 2001
From: Xiang <XiangCZ929@gmail.com>
Date: Tue, 19 Sep 2023 04:11:22 +0800
Subject: [PATCH 28/41] Fix applying rewrite parameter (#97642)

Fix applying rewrite parameter for fuzzy queries on string fields
---
 docs/changelog/97642.yaml                     |  5 ++++
 .../index/mapper/StringFieldType.java         | 23 +++++++++++++------
 .../index/mapper/TextFieldTypeTests.java      |  7 ++++++
 3 files changed, 28 insertions(+), 7 deletions(-)
 create mode 100644 docs/changelog/97642.yaml

diff --git a/docs/changelog/97642.yaml b/docs/changelog/97642.yaml
new file mode 100644
index 0000000000000..cf519e04e2d38
--- /dev/null
+++ b/docs/changelog/97642.yaml
@@ -0,0 +1,5 @@
+pr: 97642
+summary: fix fuzzy query rewrite parameter not work
+area: Search
+type: bug
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
index ab1bdd4ca5d86..6f68c2f67bdcd 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
@@ -70,13 +70,22 @@ public Query fuzzyQuery(
             );
         }
         failIfNotIndexed();
-        return new FuzzyQuery(
-            new Term(name(), indexedValueForSearch(value)),
-            fuzziness.asDistance(BytesRefs.toString(value)),
-            prefixLength,
-            maxExpansions,
-            transpositions
-        );
+        return rewriteMethod == null
+            ? new FuzzyQuery(
+                new Term(name(), indexedValueForSearch(value)),
+                fuzziness.asDistance(BytesRefs.toString(value)),
+                prefixLength,
+                maxExpansions,
+                transpositions
+            )
+            : new FuzzyQuery(
+                new Term(name(), indexedValueForSearch(value)),
+                fuzziness.asDistance(BytesRefs.toString(value)),
+                prefixLength,
+                maxExpansions,
+                transpositions,
+                rewriteMethod
+            );
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java
index 1b175a388090f..bfe61c6a14736 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.RegexpQuery;
@@ -141,6 +142,12 @@ public void testFuzzyQuery() {
             )
         );
         assertEquals("[fuzzy] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", ee.getMessage());
+
+        assertEquals(
+            new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true, MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE),
+            ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, MOCK_CONTEXT, MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE)
+        );
+
     }
 
     public void testIndexPrefixes() {

From ee39083f7199960eee5614ca555ee86810eacdce Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 18 Sep 2023 13:41:52 -0700
Subject: [PATCH 29/41] Remove es.serverless flag (#99603)

This commit removes the remaining uses of the es.serverless flag. It is
no longer needed given pluggability of the rest and cat cations.
---
 .../elasticsearch/action/ActionModule.java    |  3 --
 .../cluster/node/DiscoveryNode.java           | 10 ------
 .../cluster/node/DiscoveryNodeRole.java       | 19 ------------
 .../xpack/security/Security.java              |  3 +-
 .../xpack/security/SecurityTests.java         | 31 -------------------
 5 files changed, 1 insertion(+), 65 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 31355aac21d67..f0b755f1108cd 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -274,7 +274,6 @@
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.NamedRegistry;
@@ -509,7 +508,6 @@ public class ActionModule extends AbstractModule {
     private final RequestValidators<IndicesAliasesRequest> indicesAliasesRequestRequestValidators;
     private final ThreadPool threadPool;
     private final ReservedClusterStateService reservedClusterStateService;
-    private final boolean serverlessEnabled;
     private final RestExtension restExtension;
 
     public ActionModule(
@@ -536,7 +534,6 @@ public ActionModule(
         this.settingsFilter = settingsFilter;
         this.actionPlugins = actionPlugins;
         this.threadPool = threadPool;
-        this.serverlessEnabled = DiscoveryNode.isServerless();
         actions = setupActions(actionPlugins);
         actionFilters = setupActionFilters(actionPlugins);
         autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, systemIndices);
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
index 31cbe3b0763e1..ff5496aaef83e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
@@ -58,16 +58,6 @@ public static boolean isStateless(final Settings settings) {
         return settings.getAsBoolean(STATELESS_ENABLED_SETTING_NAME, false);
     }
 
-    /**
-     * Check if the serverless feature flag is present and set to {@code true}, indicating that the node is
-     * part of a serverless deployment.
-     *
-     * @return true if the serverless feature flag is present and set
-     */
-    public static boolean isServerless() {
-        return DiscoveryNodeRole.hasServerlessFeatureFlag();
-    }
-
     static final String COORDINATING_ONLY = "coordinating_only";
     public static final TransportVersion EXTERNAL_ID_VERSION = TransportVersions.V_8_3_0;
     public static final Comparator<DiscoveryNode> DISCOVERY_NODE_COMPARATOR = Comparator.comparing(DiscoveryNode::getName)
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
index 6107f23b57313..f295cbc9556b2 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
@@ -8,10 +8,8 @@
 
 package org.elasticsearch.cluster.node;
 
-import org.elasticsearch.Build;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.core.Booleans;
 
 import java.lang.reflect.Field;
 import java.util.Arrays;
@@ -28,19 +26,6 @@
  */
 public class DiscoveryNodeRole implements Comparable<DiscoveryNodeRole> {
 
-    /**
-     * A feature flag to indicate if serverless is available or not. Defaults to false.
-     */
-    private static final String USE_SERVERLESS_SYSTEM_PROPERTY = "es.serverless";
-    private static final Boolean USE_SERVERLESS_FEATURE_FLAG;
-    static {
-        final Boolean useStateless = Booleans.parseBoolean(System.getProperty(USE_SERVERLESS_SYSTEM_PROPERTY), false);
-        if (useStateless && Build.current().isSnapshot() == false) {
-            throw new IllegalArgumentException("Enabling serverless usage is only supported in snapshot builds");
-        }
-        USE_SERVERLESS_FEATURE_FLAG = useStateless;
-    }
-
     private final String roleName;
 
     /**
@@ -371,8 +356,4 @@ public static Optional<DiscoveryNodeRole> maybeGetRoleFromRoleName(final String
     public static DiscoveryNodeRole getRoleFromRoleName(final String roleName) {
         return maybeGetRoleFromRoleName(roleName).orElseThrow(() -> new IllegalArgumentException("unknown role [" + roleName + "]"));
     }
-
-    public static boolean hasServerlessFeatureFlag() {
-        return USE_SERVERLESS_FEATURE_FLAG;
-    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index c8a302249f023..bf4f72fbb8a98 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -910,8 +910,7 @@ Collection<Object> createComponents(
 
         final AuthenticationFailureHandler failureHandler = createAuthenticationFailureHandler(realms, extensionComponents);
 
-        // operator privileges are enabled either explicitly via the setting or if running serverless
-        final boolean operatorPrivilegesEnabled = OPERATOR_PRIVILEGES_ENABLED.get(settings) || DiscoveryNode.isServerless();
+        final boolean operatorPrivilegesEnabled = OPERATOR_PRIVILEGES_ENABLED.get(settings);
 
         if (operatorPrivilegesEnabled) {
             logger.info("operator privileges are enabled");
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
index fa7ff7b12994c..7f174ee26cf0b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
@@ -924,7 +924,6 @@ public <T> List<T> loadExtensions(Class<T> extensionPointType) {
     }
 
     public void testLoadExtensionsWhenOperatorPrivsAreDisabled() throws Exception {
-        assumeFalse("feature flag for serverless is expected to be false", DiscoveryNode.isServerless());
         Settings.Builder settingsBuilder = Settings.builder().put("xpack.security.enabled", true).put("path.home", createTempDir());
 
         if (randomBoolean()) {
@@ -951,36 +950,6 @@ public <T> List<T> loadExtensions(Class<T> extensionPointType) {
         assertThat(operatorPrivilegesService, is(NOOP_OPERATOR_PRIVILEGES_SERVICE));
     }
 
-    public void testLoadExtensionsWhenOperatorPrivsAreDisabledAndServerless() throws Exception {
-        assumeTrue("feature flag for serverless is expected to be true", DiscoveryNode.isServerless());
-        Settings.Builder settingsBuilder = Settings.builder().put("xpack.security.enabled", true).put("path.home", createTempDir());
-
-        if (randomBoolean()) {
-            settingsBuilder.put(OPERATOR_PRIVILEGES_ENABLED.getKey(), false); // doesn't matter if explicit or implicitly disabled
-        }
-
-        Settings settings = settingsBuilder.build();
-        constructNewSecurityObject(settings);
-        security.loadExtensions(new ExtensiblePlugin.ExtensionLoader() {
-            @Override
-            @SuppressWarnings("unchecked")
-            public <T> List<T> loadExtensions(Class<T> extensionPointType) {
-                List<Object> extensions = new ArrayList<>();
-                if (extensionPointType == OperatorOnlyRegistry.class) {
-                    if (randomBoolean()) {
-                        extensions.add(new DummyOperatorOnlyRegistry()); // will be used
-                    }
-                }
-                return (List<T>) extensions;
-            }
-        });
-        createComponentsUtil(settings);
-        OperatorPrivileges.OperatorPrivilegesService operatorPrivilegesService = security.getOperatorPrivilegesService();
-        OperatorOnlyRegistry registry = ((OperatorPrivileges.DefaultOperatorPrivilegesService) operatorPrivilegesService)
-            .getOperatorOnlyRegistry();
-        assertThat(registry, instanceOf(DummyOperatorOnlyRegistry.class));
-    }
-
     private void verifyHasAuthenticationHeaderValue(Exception e, String... expectedValues) {
         assertThat(e, instanceOf(ElasticsearchSecurityException.class));
         assertThat(((ElasticsearchSecurityException) e).getHeader("WWW-Authenticate"), notNullValue());

From 91e4662f133efefd8188ff966a907ca031baa790 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 18 Sep 2023 13:42:14 -0700
Subject: [PATCH 30/41] Determine latest index version dynamically (#99599)

This commit reduces some of the boilerplate needed for defining new
IndexVersion constants. Previously defining a new constant meant both
adding the constant and updating the definition of CURRENT. With this
change the CURRENT is automatically figured out. It utilizes the already
existing sorted map that we have for finding duplicate index version
definitions.
---
 .../org/elasticsearch/index/IndexVersion.java | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersion.java b/server/src/main/java/org/elasticsearch/index/IndexVersion.java
index ef238d1fe87f0..49e2172b2f4b2 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexVersion.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexVersion.java
@@ -152,17 +152,17 @@ private static IndexVersion registerIndexVersion(int id, Version luceneVersion,
      */
 
     private static class CurrentHolder {
-        private static final IndexVersion CURRENT = findCurrent(V_8_500_001);
+        private static final IndexVersion CURRENT = findCurrent();
 
         // finds the pluggable current version, or uses the given fallback
-        private static IndexVersion findCurrent(IndexVersion fallback) {
+        private static IndexVersion findCurrent() {
             var versionExtension = ExtensionLoader.loadSingleton(ServiceLoader.load(VersionExtension.class), () -> null);
             if (versionExtension == null) {
-                return fallback;
+                return LATEST_DEFINED;
             }
             var version = versionExtension.getCurrentIndexVersion();
 
-            assert version.onOrAfter(fallback);
+            assert version.onOrAfter(LATEST_DEFINED);
             assert version.luceneVersion.equals(Version.LATEST)
                 : "IndexVersion must be upgraded to ["
                 + Version.LATEST
@@ -175,7 +175,12 @@ private static IndexVersion findCurrent(IndexVersion fallback) {
 
     public static final IndexVersion MINIMUM_COMPATIBLE = V_7_0_0;
 
+    private static final NavigableMap<Integer, IndexVersion> VERSION_IDS = getAllVersionIds(IndexVersion.class);
+
+    static final IndexVersion LATEST_DEFINED;
     static {
+        LATEST_DEFINED = VERSION_IDS.lastEntry().getValue();
+
         // see comment on IDS field
         // now we're registered the index versions, we can clear the map
         IDS = null;
@@ -220,12 +225,6 @@ static NavigableMap<Integer, IndexVersion> getAllVersionIds(Class<?> cls) {
         return Collections.unmodifiableNavigableMap(builder);
     }
 
-    private static final NavigableMap<Integer, IndexVersion> VERSION_IDS;
-
-    static {
-        VERSION_IDS = getAllVersionIds(IndexVersion.class);
-    }
-
     static Collection<IndexVersion> getAllVersions() {
         return VERSION_IDS.values();
     }

From 9ce92a94a5894074e4ccf79e0b6c41ccd8a94170 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 18 Sep 2023 14:10:14 -0700
Subject: [PATCH 31/41] Fix QuantileStates for non-existent group (#99558)

OrdinalsGroupingOperator can retrieve the state of a group that doesn't
exist in the grouping state. The grouping state needs to return null in
this case. However, QuantileStates doesn't follow this.
---
 ...dianAbsoluteDeviationDoubleAggregator.java |  2 +-
 .../MedianAbsoluteDeviationIntAggregator.java |  2 +-
 ...MedianAbsoluteDeviationLongAggregator.java |  2 +-
 .../PercentileDoubleAggregator.java           |  2 +-
 .../aggregation/PercentileIntAggregator.java  |  2 +-
 .../aggregation/PercentileLongAggregator.java |  2 +-
 .../compute/aggregation/QuantileStates.java   | 10 +++++--
 .../xpack/esql/action/EsqlActionIT.java       | 29 +++++++++++++++++++
 8 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregator.java
index b2b30521ee3a4..399207fa3a30e 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregator.java
@@ -57,7 +57,7 @@ public static void combineStates(
         QuantileStates.GroupingState state,
         int statePosition
     ) {
-        current.add(currentGroupId, state.get(statePosition));
+        current.add(currentGroupId, state.getOrNull(statePosition));
     }
 
     public static Block evaluateFinal(QuantileStates.GroupingState state, IntVector selected) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregator.java
index 43032c057d769..3275cdfc3fcac 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregator.java
@@ -57,7 +57,7 @@ public static void combineStates(
         QuantileStates.GroupingState state,
         int statePosition
     ) {
-        current.add(currentGroupId, state.get(statePosition));
+        current.add(currentGroupId, state.getOrNull(statePosition));
     }
 
     public static Block evaluateFinal(QuantileStates.GroupingState state, IntVector selected) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregator.java
index 11c0218b1728e..3183fc2719b99 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregator.java
@@ -57,7 +57,7 @@ public static void combineStates(
         QuantileStates.GroupingState state,
         int statePosition
     ) {
-        current.add(currentGroupId, state.get(statePosition));
+        current.add(currentGroupId, state.getOrNull(statePosition));
     }
 
     public static Block evaluateFinal(QuantileStates.GroupingState state, IntVector selected) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileDoubleAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileDoubleAggregator.java
index 23e82f27602e0..a8e64bb8d959a 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileDoubleAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileDoubleAggregator.java
@@ -57,7 +57,7 @@ public static void combineStates(
         QuantileStates.GroupingState state,
         int statePosition
     ) {
-        current.add(currentGroupId, state.get(statePosition));
+        current.add(currentGroupId, state.getOrNull(statePosition));
     }
 
     public static Block evaluateFinal(QuantileStates.GroupingState state, IntVector selected) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileIntAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileIntAggregator.java
index 7f319786454f3..3f8f1bc1c5882 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileIntAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileIntAggregator.java
@@ -57,7 +57,7 @@ public static void combineStates(
         QuantileStates.GroupingState state,
         int statePosition
     ) {
-        current.add(currentGroupId, state.get(statePosition));
+        current.add(currentGroupId, state.getOrNull(statePosition));
     }
 
     public static Block evaluateFinal(QuantileStates.GroupingState state, IntVector selected) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileLongAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileLongAggregator.java
index 4611153bc97bf..7905d7f37a152 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileLongAggregator.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileLongAggregator.java
@@ -57,7 +57,7 @@ public static void combineStates(
         QuantileStates.GroupingState state,
         int statePosition
     ) {
-        current.add(currentGroupId, state.get(statePosition));
+        current.add(currentGroupId, state.getOrNull(statePosition));
     }
 
     public static Block evaluateFinal(QuantileStates.GroupingState state, IntVector selectedGroups) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java
index be47e346abc04..77da2c8a03817 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java
@@ -148,8 +148,12 @@ void add(int groupId, BytesRef other) {
             getOrAddGroup(groupId).add(deserializeDigest(other));
         }
 
-        TDigestState get(int position) {
-            return digests.get(position);
+        TDigestState getOrNull(int position) {
+            if (position < digests.size()) {
+                return digests.get(position);
+            } else {
+                return null;
+            }
         }
 
         /** Extracts an intermediate view of the contents of this state.  */
@@ -161,7 +165,7 @@ public void toIntermediate(Block[] blocks, int offset, IntVector selected) {
                 int group = selected.getInt(i);
                 TDigestState state;
                 if (group < digests.size()) {
-                    state = get(group);
+                    state = getOrNull(group);
                     if (state == null) {
                         state = TDigestState.create(DEFAULT_COMPRESSION);
                     }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java
index fdc83161dcd06..a001ece05725a 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java
@@ -37,6 +37,7 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.OptionalDouble;
 import java.util.concurrent.CountDownLatch;
@@ -1053,6 +1054,34 @@ public void testReturnNoNestedDocuments() throws IOException, ExecutionException
         assertNoNestedDocuments("from " + alias + " | where data >= 50", Arrays.stream(countValuesGreaterThanFifty).sum(), 50L, 100L);
     }
 
+    public void testGroupingMultiValueByOrdinals() {
+        String indexName = "test-ordinals";
+        assertAcked(client().admin().indices().prepareCreate(indexName).setMapping("kw", "type=keyword", "v", "type=long").get());
+        int numDocs = randomIntBetween(10, 200);
+        for (int i = 0; i < numDocs; i++) {
+            Map<String, Object> source = new HashMap<>();
+            source.put("kw", "key-" + randomIntBetween(1, 20));
+            List<Integer> values = new ArrayList<>();
+            int numValues = between(0, 2);
+            for (int v = 0; v < numValues; v++) {
+                values.add(randomIntBetween(1, 1000));
+            }
+            if (values.isEmpty() == false) {
+                source.put("v", values);
+            }
+            client().prepareIndex(indexName).setSource(source).get();
+            if (randomInt(100) < 20) {
+                client().admin().indices().prepareRefresh(indexName).get();
+            }
+        }
+        client().admin().indices().prepareRefresh(indexName).get();
+        var functions = List.of("min(v)", "max(v)", "count_distinct(v)", "count(v)", "sum(v)", "avg(v)", "percentile(v, 90)");
+        for (String fn : functions) {
+            String query = String.format(Locale.ROOT, "from %s | stats s = %s by kw", indexName, fn);
+            run(query);
+        }
+    }
+
     private void createNestedMappingIndex(String indexName) throws IOException {
         XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();

From 92458fcbcca2d36a2a16f98e1f91d739bbebbb98 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 19 Sep 2023 06:24:46 +0100
Subject: [PATCH 32/41] Small cleanup in BlobStoreRepository#finalizeSnapshot
 (#99635)

Reorders the operations into their logical order and adds a few TODOs
for gaps that this cleanup exposes.
---
 .../blobstore/BlobStoreSizeLimitIT.java       |   9 +-
 .../blobstore/BlobStoreRepository.java        | 211 +++++++++++-------
 2 files changed, 138 insertions(+), 82 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreSizeLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreSizeLimitIT.java
index 73ad2737f1f10..ffe6133e034bc 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreSizeLimitIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreSizeLimitIT.java
@@ -14,11 +14,14 @@
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
+import org.elasticsearch.snapshots.SnapshotException;
 import org.hamcrest.Matchers;
 
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class BlobStoreSizeLimitIT extends AbstractSnapshotIntegTestCase {
 
@@ -32,7 +35,11 @@ public void testBlobStoreSizeIsLimited() throws Exception {
         );
         final List<String> snapshotNames = createNSnapshots(repoName, maxSnapshots);
         final ActionFuture<CreateSnapshotResponse> failingSnapshotFuture = startFullSnapshot(repoName, "failing-snapshot");
-        final RepositoryException repositoryException = expectThrows(RepositoryException.class, failingSnapshotFuture::actionGet);
+        final SnapshotException snapshotException = expectThrows(SnapshotException.class, failingSnapshotFuture::actionGet);
+        assertThat(snapshotException.getRepositoryName(), equalTo(repoName));
+        assertThat(snapshotException.getSnapshotName(), equalTo("failing-snapshot"));
+        assertThat(snapshotException.getCause(), instanceOf(RepositoryException.class));
+        final RepositoryException repositoryException = (RepositoryException) snapshotException.getCause();
         assertThat(
             repositoryException.getMessage(),
             Matchers.endsWith(
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index afe5562103240..436d34bc39c21 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.RefCountingListener;
 import org.elasticsearch.action.support.RefCountingRunnable;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
@@ -1326,44 +1327,107 @@ public void finalizeSnapshot(final FinalizeSnapshotContext finalizeSnapshotConte
         // when writing the index-${N} to each shard directory.
         final IndexVersion repositoryMetaVersion = finalizeSnapshotContext.repositoryMetaVersion();
         final boolean writeShardGens = SnapshotsService.useShardGenerations(repositoryMetaVersion);
-        final Consumer<Exception> onUpdateFailure = e -> finalizeSnapshotContext.onFailure(
-            new SnapshotException(metadata.name(), snapshotId, "failed to update snapshot in repository", e)
-        );
 
         final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
 
         final boolean writeIndexGens = SnapshotsService.useIndexGenerations(repositoryMetaVersion);
 
-        final ListenableFuture<RepositoryData> repoDataListener = new ListenableFuture<>();
-        getRepositoryData(repoDataListener);
-        repoDataListener.addListener(ActionListener.wrap(existingRepositoryData -> {
-            final int existingSnapshotCount = existingRepositoryData.getSnapshotIds().size();
-            if (existingSnapshotCount >= maxSnapshotCount) {
-                finalizeSnapshotContext.onFailure(
-                    new RepositoryException(
+        record MetadataWriteResult(
+            RepositoryData existingRepositoryData,
+            Map<IndexId, String> indexMetas,
+            Map<String, String> indexMetaIdentifiers
+        ) {}
+
+        record RootBlobUpdateResult(RepositoryData oldRepositoryData, RepositoryData newRepositoryData) {}
+
+        SubscribableListener
+
+            // Get the current RepositoryData
+            .<RepositoryData>newForked(this::getRepositoryData)
+
+            // Identify and write the missing metadata
+            .<MetadataWriteResult>andThen((l, existingRepositoryData) -> {
+                final int existingSnapshotCount = existingRepositoryData.getSnapshotIds().size();
+                if (existingSnapshotCount >= maxSnapshotCount) {
+                    throw new RepositoryException(
                         metadata.name(),
-                        "Cannot add another snapshot to this repository as it "
-                            + "already contains ["
+                        "Cannot add another snapshot to this repository as it already contains ["
                             + existingSnapshotCount
                             + "] snapshots and is configured to hold up to ["
                             + maxSnapshotCount
                             + "] snapshots only."
-                    )
-                );
-                return;
-            }
+                    );
+                }
 
-            final Map<IndexId, String> indexMetas;
-            final Map<String, String> indexMetaIdentifiers;
-            if (writeIndexGens) {
-                indexMetaIdentifiers = ConcurrentCollections.newConcurrentMap();
-                indexMetas = ConcurrentCollections.newConcurrentMap();
-            } else {
-                indexMetas = null;
-                indexMetaIdentifiers = null;
-            }
+                final MetadataWriteResult metadataWriteResult;
+                if (writeIndexGens) {
+                    metadataWriteResult = new MetadataWriteResult(
+                        existingRepositoryData,
+                        ConcurrentCollections.newConcurrentMap(),
+                        ConcurrentCollections.newConcurrentMap()
+                    );
+                } else {
+                    metadataWriteResult = new MetadataWriteResult(existingRepositoryData, null, null);
+                }
+
+                try (var allMetaListeners = new RefCountingListener(l.map(ignored -> metadataWriteResult))) {
+                    // We ignore all FileAlreadyExistsException when writing metadata since otherwise a master failover while in this method
+                    // will mean that no snap-${uuid}.dat blob is ever written for this snapshot. This is safe because any updated version
+                    // of the index or global metadata will be compatible with the segments written in this snapshot as well.
+                    // Failing on an already existing index-${repoGeneration} below ensures that the index.latest blob is not updated in a
+                    // way that decrements the generation it points at
+
+                    // Write global metadata
+                    final Metadata clusterMetadata = finalizeSnapshotContext.clusterMetadata();
+                    executor.execute(
+                        ActionRunnable.run(
+                            allMetaListeners.acquire(),
+                            () -> GLOBAL_METADATA_FORMAT.write(clusterMetadata, blobContainer(), snapshotId.getUUID(), compress)
+                        )
+                    );
+
+                    // Write the index metadata for each index in the snapshot
+                    for (IndexId index : indices) {
+                        executor.execute(ActionRunnable.run(allMetaListeners.acquire(), () -> {
+                            final IndexMetadata indexMetaData = clusterMetadata.index(index.getName());
+                            if (writeIndexGens) {
+                                final String identifiers = IndexMetaDataGenerations.buildUniqueIdentifier(indexMetaData);
+                                String metaUUID = existingRepositoryData.indexMetaDataGenerations().getIndexMetaBlobId(identifiers);
+                                if (metaUUID == null) {
+                                    // We don't yet have this version of the metadata so we write it
+                                    metaUUID = UUIDs.base64UUID();
+                                    INDEX_METADATA_FORMAT.write(indexMetaData, indexContainer(index), metaUUID, compress);
+                                    metadataWriteResult.indexMetaIdentifiers().put(identifiers, metaUUID);
+                                } // else this task was largely a no-op - TODO no need to fork in that case
+                                metadataWriteResult.indexMetas().put(index, identifiers);
+                            } else {
+                                INDEX_METADATA_FORMAT.write(
+                                    clusterMetadata.index(index.getName()),
+                                    indexContainer(index),
+                                    snapshotId.getUUID(),
+                                    compress
+                                );
+                            }
+                        }));
+                    }
+
+                    // Write the SnapshotInfo blob
+                    executor.execute(
+                        ActionRunnable.run(
+                            allMetaListeners.acquire(),
+                            () -> SNAPSHOT_FORMAT.write(snapshotInfo, blobContainer(), snapshotId.getUUID(), compress)
+                        )
+                    );
+
+                    // TODO fail fast if any metadata write fails
+                    // TODO clean up successful metadata writes on failure (needs care, we must not clobber another node concurrently
+                    // finalizing the same snapshot: we can only clean up after removing the failed snapshot from the cluster state)
+                }
+            })
 
-            try (var allMetaListeners = new RefCountingListener(ActionListener.wrap(v -> {
+            // Update the root blob
+            .<RootBlobUpdateResult>andThen((l, metadataWriteResult) -> {
+                // unlikely, but in theory we could still be on the thread which called finalizeSnapshot - TODO must fork to SNAPSHOT here
                 final String slmPolicy = slmPolicy(snapshotInfo);
                 final SnapshotDetails snapshotDetails = new SnapshotDetails(
                     snapshotInfo.state(),
@@ -1372,64 +1436,41 @@ public void finalizeSnapshot(final FinalizeSnapshotContext finalizeSnapshotConte
                     snapshotInfo.endTime(),
                     slmPolicy
                 );
+                final var existingRepositoryData = metadataWriteResult.existingRepositoryData();
                 writeIndexGen(
-                    existingRepositoryData.addSnapshot(snapshotId, snapshotDetails, shardGenerations, indexMetas, indexMetaIdentifiers),
+                    existingRepositoryData.addSnapshot(
+                        snapshotId,
+                        snapshotDetails,
+                        shardGenerations,
+                        metadataWriteResult.indexMetas(),
+                        metadataWriteResult.indexMetaIdentifiers()
+                    ),
                     repositoryStateId,
                     repositoryMetaVersion,
                     finalizeSnapshotContext::updatedClusterState,
-                    ActionListener.wrap(newRepoData -> {
-                        finalizeSnapshotContext.onResponse(newRepoData);
-                        cleanupOldMetadata(existingRepositoryData, newRepoData, finalizeSnapshotContext, snapshotInfo, writeShardGens);
-                    }, onUpdateFailure)
+                    l.map(newRepositoryData -> new RootBlobUpdateResult(existingRepositoryData, newRepositoryData))
                 );
-            }, onUpdateFailure))) {
-
-                // We ignore all FileAlreadyExistsException when writing metadata since otherwise a master failover while in this method
-                // will mean that no snap-${uuid}.dat blob is ever written for this snapshot. This is safe because any updated version of
-                // the index or global metadata will be compatible with the segments written in this snapshot as well.
-                // Failing on an already existing index-${repoGeneration} below ensures that the index.latest blob is not updated in a way
-                // that decrements the generation it points at
-                final Metadata clusterMetadata = finalizeSnapshotContext.clusterMetadata();
-                // Write Global MetaData
-                executor.execute(
-                    ActionRunnable.run(
-                        allMetaListeners.acquire(),
-                        () -> GLOBAL_METADATA_FORMAT.write(clusterMetadata, blobContainer(), snapshotId.getUUID(), compress)
-                    )
+                // NB failure of writeIndexGen doesn't guarantee the update failed, so we cannot safely clean anything up on failure
+            })
+
+            // Report success, then clean up.
+            .<RepositoryData>andThen((l, rootBlobUpdateResult) -> {
+                l.onResponse(rootBlobUpdateResult.newRepositoryData());
+                cleanupOldMetadata(
+                    rootBlobUpdateResult.oldRepositoryData(),
+                    rootBlobUpdateResult.newRepositoryData(),
+                    finalizeSnapshotContext,
+                    snapshotInfo,
+                    writeShardGens
                 );
+            })
 
-                // write the index metadata for each index in the snapshot
-                for (IndexId index : indices) {
-                    executor.execute(ActionRunnable.run(allMetaListeners.acquire(), () -> {
-                        final IndexMetadata indexMetaData = clusterMetadata.index(index.getName());
-                        if (writeIndexGens) {
-                            final String identifiers = IndexMetaDataGenerations.buildUniqueIdentifier(indexMetaData);
-                            String metaUUID = existingRepositoryData.indexMetaDataGenerations().getIndexMetaBlobId(identifiers);
-                            if (metaUUID == null) {
-                                // We don't yet have this version of the metadata so we write it
-                                metaUUID = UUIDs.base64UUID();
-                                INDEX_METADATA_FORMAT.write(indexMetaData, indexContainer(index), metaUUID, compress);
-                                indexMetaIdentifiers.put(identifiers, metaUUID);
-                            }
-                            indexMetas.put(index, identifiers);
-                        } else {
-                            INDEX_METADATA_FORMAT.write(
-                                clusterMetadata.index(index.getName()),
-                                indexContainer(index),
-                                snapshotId.getUUID(),
-                                compress
-                            );
-                        }
-                    }));
-                }
-                executor.execute(
-                    ActionRunnable.run(
-                        allMetaListeners.acquire(),
-                        () -> SNAPSHOT_FORMAT.write(snapshotInfo, blobContainer(), snapshotId.getUUID(), compress)
-                    )
-                );
-            }
-        }, onUpdateFailure));
+            // Finally subscribe the context as the listener, wrapping exceptions if needed
+            .addListener(
+                finalizeSnapshotContext.delegateResponse(
+                    (l, e) -> l.onFailure(new SnapshotException(metadata.name(), snapshotId, "failed to update snapshot in repository", e))
+                )
+            );
     }
 
     // Delete all old shard gen and root level index blobs that aren't referenced any longer as a result from moving to updated
@@ -1471,13 +1512,21 @@ private void cleanupOldMetadata(
                 }
             }
         }
+
         if (toDelete.isEmpty() == false) {
-            threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
-                try {
+            threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new AbstractRunnable() {
+                @Override
+                protected void doRun() throws Exception {
                     deleteFromContainer(blobContainer(), toDelete.iterator());
-                } catch (Exception e) {
+                }
+
+                @Override
+                public void onFailure(Exception e) {
                     logger.warn("Failed to clean up old metadata blobs", e);
-                } finally {
+                }
+
+                @Override
+                public void onAfter() {
                     finalizeSnapshotContext.onDone(snapshotInfo);
                 }
             });

From 4bc1afddda9a7475299229a5458f4d659bec88c4 Mon Sep 17 00:00:00 2001
From: Ignacio Vera <ivera@apache.org>
Date: Tue, 19 Sep 2023 09:46:51 +0200
Subject: [PATCH 33/41] Move Aggregator#buildTopLevel() to search worker
 thread. (#98715)

This commit introduces an AggregatorCollector that contains a finish method which performs aggregation
post-collection and builds the internal aggregation for this collector. This method is called on the worker
thread at the end of the collection phase.
---
 docs/reference/search/profile.asciidoc        |   2 +-
 .../search/aggregations/AggregationPhase.java |  83 +++-----------
 .../aggregations/AggregatorCollector.java     |  61 +++++++++++
 .../AggregatorCollectorManager.java           |  55 ++++++++++
 .../SearchContextAggregations.java            |  24 +----
 .../GlobalOrdinalsStringTermsAggregator.java  |  36 ++++---
 .../SignificantTermsAggregationBuilder.java   |   5 +
 .../SignificantTermsAggregatorFactory.java    |   2 +-
 .../StringTermsAggregatorFromFilters.java     |  25 +++--
 .../bucket/terms/TermsAggregatorFactory.java  |   6 +-
 .../support/AggregationContext.java           |   4 +-
 .../search/internal/ContextIndexSearcher.java |  25 ++++-
 .../query/InternalProfileCollector.java       |   2 +-
 .../search/query/QueryPhase.java              |  51 ++++-----
 .../query/QueryPhaseCollectorManager.java     |  25 +++--
 .../aggregations/AggregatorTestCase.java      | 102 +++++++++---------
 .../topmetrics/TopMetricsAggregatorTests.java |   1 +
 .../mr/ItemSetMapReduceValueSource.java       |   8 +-
 18 files changed, 293 insertions(+), 224 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/AggregatorCollector.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/AggregatorCollectorManager.java

diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
index 7f9cf34ba7dad..52dfb91475c53 100644
--- a/docs/reference/search/profile.asciidoc
+++ b/docs/reference/search/profile.asciidoc
@@ -740,7 +740,7 @@ The API returns the following result:
                     "time_in_nanos": 22577
                   },
                   {
-                    "name": "BucketCollectorWrapper: [BucketCollectorWrapper[bucketCollector=[my_scoped_agg, my_global_agg]]]",
+                    "name": "AggregatorCollector: [my_scoped_agg, my_global_agg]",
                     "reason": "aggregation",
                     "time_in_nanos": 867617
                   }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
index 786fabca78d63..ce810507f3082 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
@@ -7,8 +7,6 @@
  */
 package org.elasticsearch.search.aggregations;
 
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.CollectorManager;
 import org.elasticsearch.action.search.SearchShardTask;
 import org.elasticsearch.search.aggregations.support.TimeSeriesIndexSearcher;
 import org.elasticsearch.search.internal.SearchContext;
@@ -16,7 +14,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.function.Supplier;
 
@@ -31,36 +28,30 @@ public static void preProcess(SearchContext context) {
         if (context.aggregations() == null) {
             return;
         }
-        final Supplier<Collector> collectorSupplier;
+        final Supplier<AggregatorCollector> collectorSupplier;
         if (context.aggregations().isInSortOrderExecutionRequired()) {
-            executeInSortOrder(context, newBucketCollector(context));
-            collectorSupplier = () -> BucketCollector.NO_OP_COLLECTOR;
+            AggregatorCollector collector = newAggregatorCollector(context);
+            executeInSortOrder(context, collector.bucketCollector);
+            collectorSupplier = () -> new AggregatorCollector(collector.aggregators, BucketCollector.NO_OP_BUCKET_COLLECTOR);
         } else {
-            collectorSupplier = () -> newBucketCollector(context).asCollector();
+            collectorSupplier = () -> newAggregatorCollector(context);
         }
-        context.aggregations().registerAggsCollectorManager(new CollectorManager<>() {
-            @Override
-            public Collector newCollector() {
-                return collectorSupplier.get();
-            }
-
-            @Override
-            public Void reduce(Collection<Collector> collectors) {
-                // we cannot run post-collection method here because we need to do it after the optional timeout
-                // has been removed from the index searcher. Therefore, we delay this processing to the
-                // AggregationPhase#execute method.
-                return null;
-            }
-        });
+        context.aggregations()
+            .registerAggsCollectorManager(
+                new AggregatorCollectorManager(
+                    collectorSupplier,
+                    internalAggregations -> context.queryResult().aggregations(internalAggregations),
+                    () -> context.aggregations().getAggregationReduceContextBuilder().forPartialReduction()
+                )
+            );
     }
 
-    private static BucketCollector newBucketCollector(SearchContext context) {
+    private static AggregatorCollector newAggregatorCollector(SearchContext context) {
         try {
             Aggregator[] aggregators = context.aggregations().factories().createTopLevelAggregators();
-            context.aggregations().aggregators(aggregators);
             BucketCollector bucketCollector = MultiBucketCollector.wrap(true, List.of(aggregators));
             bucketCollector.preCollection();
-            return bucketCollector;
+            return new AggregatorCollector(aggregators, bucketCollector);
         } catch (IOException e) {
             throw new AggregationInitializationException("Could not initialize aggregators", e);
         }
@@ -96,48 +87,4 @@ private static List<Runnable> getCancellationChecks(SearchContext context) {
 
         return cancellationChecks;
     }
-
-    public static void execute(SearchContext context) {
-        if (context.aggregations() == null) {
-            context.queryResult().aggregations(null);
-            return;
-        }
-
-        if (context.queryResult().hasAggs()) {
-            // no need to compute the aggs twice, they should be computed on a per context basis
-            return;
-        }
-
-        final List<InternalAggregations> internalAggregations = new ArrayList<>(context.aggregations().aggregators().size());
-        for (Aggregator[] aggregators : context.aggregations().aggregators()) {
-            final List<InternalAggregation> aggregations = new ArrayList<>(aggregators.length);
-            for (Aggregator aggregator : aggregators) {
-                try {
-                    aggregations.add(aggregator.buildTopLevel());
-                } catch (IOException e) {
-                    throw new AggregationExecutionException("Failed to build aggregation [" + aggregator.name() + "]", e);
-                }
-                // release the aggregator to claim the used bytes as we don't need it anymore
-                aggregator.releaseAggregations();
-            }
-            internalAggregations.add(InternalAggregations.from(aggregations));
-        }
-
-        if (internalAggregations.size() > 1) {
-            // we execute this search using more than one slice. In order to keep memory requirements
-            // low, we do a partial reduction here.
-            context.queryResult()
-                .aggregations(
-                    InternalAggregations.topLevelReduce(
-                        internalAggregations,
-                        context.aggregations().getAggregationReduceContextBuilder().forPartialReduction()
-                    )
-                );
-        } else {
-            context.queryResult().aggregations(internalAggregations.get(0));
-        }
-
-        // disable aggregations so that they don't run on next pages in case of scrolling
-        context.aggregations(null);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorCollector.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorCollector.java
new file mode 100644
index 0000000000000..56083895d72ce
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorCollector.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.ScoreMode;
+import org.elasticsearch.search.internal.TwoPhaseCollector;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/** Collector that controls the life cycle of an aggregation document collection. */
+public class AggregatorCollector implements TwoPhaseCollector {
+    final Aggregator[] aggregators;
+    final BucketCollector bucketCollector;
+    final List<InternalAggregation> internalAggregations;
+
+    public AggregatorCollector(Aggregator[] aggregators, BucketCollector bucketCollector) {
+        this.aggregators = aggregators;
+        this.bucketCollector = bucketCollector;
+        this.internalAggregations = new ArrayList<>(aggregators.length);
+    }
+
+    @Override
+    public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+        return bucketCollector.getLeafCollector(new AggregationExecutionContext(context, null, null, null));
+    }
+
+    @Override
+    public ScoreMode scoreMode() {
+        return bucketCollector.scoreMode();
+    }
+
+    @Override
+    public void doPostCollection() throws IOException {
+        bucketCollector.postCollection();
+        for (Aggregator aggregator : aggregators) {
+            internalAggregations.add(aggregator.buildTopLevel());
+            // release the aggregator to claim the used bytes as we don't need it anymore
+            aggregator.releaseAggregations();
+        }
+    }
+
+    @Override
+    public String toString() {
+        String[] aggNames = new String[aggregators.length];
+        for (int i = 0; i < aggregators.length; i++) {
+            aggNames[i] = aggregators[i].name();
+        }
+        return Arrays.toString(aggNames);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorCollectorManager.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorCollectorManager.java
new file mode 100644
index 0000000000000..51e302ea9fdd0
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorCollectorManager.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.apache.lucene.search.CollectorManager;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+/** Collector manager that produces {@link AggregatorCollector} and merges them during the reduce phase. */
+public class AggregatorCollectorManager implements CollectorManager<AggregatorCollector, Void> {
+
+    private final Supplier<AggregatorCollector> collectorSupplier;
+    private final Consumer<InternalAggregations> internalAggregationsConsumer;
+    private final Supplier<AggregationReduceContext> reduceContextSupplier;
+
+    public AggregatorCollectorManager(
+        Supplier<AggregatorCollector> collectorSupplier,
+        Consumer<InternalAggregations> internalAggregationsConsumer,
+        Supplier<AggregationReduceContext> reduceContextSupplier
+    ) {
+        this.collectorSupplier = collectorSupplier;
+        this.internalAggregationsConsumer = internalAggregationsConsumer;
+        this.reduceContextSupplier = reduceContextSupplier;
+    }
+
+    @Override
+    public AggregatorCollector newCollector() throws IOException {
+        return collectorSupplier.get();
+    }
+
+    @Override
+    public Void reduce(Collection<AggregatorCollector> collectors) throws IOException {
+        if (collectors.size() > 1) {
+            // we execute this search using more than one slice. In order to keep memory requirements
+            // low, we do a partial reduction here.
+            final List<InternalAggregations> internalAggregations = new ArrayList<>(collectors.size());
+            collectors.forEach(c -> internalAggregations.add(InternalAggregations.from(c.internalAggregations)));
+            internalAggregationsConsumer.accept(InternalAggregations.topLevelReduce(internalAggregations, reduceContextSupplier.get()));
+        } else if (collectors.size() == 1) {
+            internalAggregationsConsumer.accept(InternalAggregations.from(collectors.iterator().next().internalAggregations));
+        }
+        return null;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java b/server/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java
index 2b7e2f49d6eff..361b9e8da557f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java
@@ -7,11 +7,8 @@
  */
 package org.elasticsearch.search.aggregations;
 
-import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.CollectorManager;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.util.function.Supplier;
 
 /**
@@ -21,8 +18,7 @@ public class SearchContextAggregations {
 
     private final AggregatorFactories factories;
     private final Supplier<AggregationReduceContext.Builder> toAggregationReduceContextBuilder;
-    private final List<Aggregator[]> aggregators;
-    private CollectorManager<Collector, Void> aggCollectorManager;
+    private CollectorManager<AggregatorCollector, Void> aggCollectorManager;
 
     /**
      * Creates a new aggregation context with the parsed aggregator factories
@@ -33,37 +29,23 @@ public SearchContextAggregations(
     ) {
         this.factories = factories;
         this.toAggregationReduceContextBuilder = toAggregationReduceContextBuilder;
-        this.aggregators = new ArrayList<>();
     }
 
     public AggregatorFactories factories() {
         return factories;
     }
 
-    public List<Aggregator[]> aggregators() {
-        return aggregators;
-    }
-
-    /**
-     * Registers all the created aggregators (top level aggregators) for the search execution context.
-     *
-     * @param aggregators The top level aggregators of the search execution.
-     */
-    public void aggregators(Aggregator[] aggregators) {
-        this.aggregators.add(aggregators);
-    }
-
     /**
      * Registers the collector to be run for the aggregations phase
      */
-    public void registerAggsCollectorManager(CollectorManager<Collector, Void> aggCollectorManager) {
+    public void registerAggsCollectorManager(CollectorManager<AggregatorCollector, Void> aggCollectorManager) {
         this.aggCollectorManager = aggCollectorManager;
     }
 
     /**
      * Returns the collector to be run for the aggregations phase
      */
-    public CollectorManager<Collector, Void> getAggsCollectorManager() {
+    public CollectorManager<AggregatorCollector, Void> getAggsCollectorManager() {
         return aggCollectorManager;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
index 3d00a8d8f10fa..070334d9287d1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PriorityQueue;
+import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.common.util.LongHash;
@@ -56,8 +57,9 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
     protected final ValuesSource.Bytes.WithOrdinals valuesSource;
 
     private final LongPredicate acceptedGlobalOrdinals;
+
+    private final CheckedSupplier<SortedSetDocValues, IOException> valuesSupplier;
     private final long valueCount;
-    private final GlobalOrdLookupFunction lookupGlobalOrd;
     protected final CollectionStrategy collectionStrategy;
     protected int segmentsWithSingleValuedOrds = 0;
     protected int segmentsWithMultiValuedOrds = 0;
@@ -71,7 +73,7 @@ public GlobalOrdinalsStringTermsAggregator(
         AggregatorFactories factories,
         Function<GlobalOrdinalsStringTermsAggregator, ResultStrategy<?, ?, ?>> resultStrategy,
         ValuesSource.Bytes.WithOrdinals valuesSource,
-        SortedSetDocValues values,
+        CheckedSupplier<SortedSetDocValues, IOException> valuesSupplier,
         BucketOrder order,
         DocValueFormat format,
         BucketCountThresholds bucketCountThresholds,
@@ -87,8 +89,8 @@ public GlobalOrdinalsStringTermsAggregator(
         super(name, factories, context, parent, order, format, bucketCountThresholds, collectionMode, showTermDocCountError, metadata);
         this.resultStrategy = resultStrategy.apply(this); // ResultStrategy needs a reference to the Aggregator to do its job.
         this.valuesSource = valuesSource;
-        this.valueCount = values.getValueCount();
-        this.lookupGlobalOrd = values::lookupOrd;
+        this.valuesSupplier = valuesSupplier;
+        this.valueCount = valuesSupplier.get().getValueCount();
         this.acceptedGlobalOrdinals = acceptedOrds;
         if (remapGlobalOrds) {
             this.collectionStrategy = new RemapGlobalOrds(cardinality);
@@ -265,7 +267,7 @@ static class LowCardinality extends GlobalOrdinalsStringTermsAggregator {
             AggregatorFactories factories,
             Function<GlobalOrdinalsStringTermsAggregator, ResultStrategy<?, ?, ?>> resultStrategy,
             ValuesSource.Bytes.WithOrdinals valuesSource,
-            SortedSetDocValues values,
+            CheckedSupplier<SortedSetDocValues, IOException> valuesSupplier,
             BucketOrder order,
             DocValueFormat format,
             BucketCountThresholds bucketCountThresholds,
@@ -281,7 +283,7 @@ static class LowCardinality extends GlobalOrdinalsStringTermsAggregator {
                 factories,
                 resultStrategy,
                 valuesSource,
-                values,
+                valuesSupplier,
                 order,
                 format,
                 bucketCountThresholds,
@@ -588,6 +590,7 @@ private InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws
 
             B[][] topBucketsPreOrd = buildTopBucketsPerOrd(owningBucketOrds.length);
             long[] otherDocCount = new long[owningBucketOrds.length];
+            GlobalOrdLookupFunction lookupGlobalOrd = valuesSupplier.get()::lookupOrd;
             for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
                 final int size;
                 if (bucketCountThresholds.getMinDocCount() == 0) {
@@ -598,7 +601,7 @@ private InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws
                 }
                 PriorityQueue<TB> ordered = buildPriorityQueue(size);
                 final int finalOrdIdx = ordIdx;
-                BucketUpdater<TB> updater = bucketUpdater(owningBucketOrds[ordIdx]);
+                BucketUpdater<TB> updater = bucketUpdater(owningBucketOrds[ordIdx], lookupGlobalOrd);
                 collectionStrategy.forEach(owningBucketOrds[ordIdx], new BucketInfoConsumer() {
                     TB spare = null;
 
@@ -618,7 +621,7 @@ public void accept(long globalOrd, long bucketOrd, long docCount) throws IOExcep
                 // Get the top buckets
                 topBucketsPreOrd[ordIdx] = buildBuckets(ordered.size());
                 for (int i = ordered.size() - 1; i >= 0; --i) {
-                    topBucketsPreOrd[ordIdx][i] = convertTempBucketToRealBucket(ordered.pop());
+                    topBucketsPreOrd[ordIdx][i] = convertTempBucketToRealBucket(ordered.pop(), lookupGlobalOrd);
                     otherDocCount[ordIdx] -= topBucketsPreOrd[ordIdx][i].getDocCount();
                 }
             }
@@ -653,7 +656,7 @@ public void accept(long globalOrd, long bucketOrd, long docCount) throws IOExcep
          * Update fields in {@code spare} to reflect information collected for
          * this bucket ordinal.
          */
-        abstract BucketUpdater<TB> bucketUpdater(long owningBucketOrd) throws IOException;
+        abstract BucketUpdater<TB> bucketUpdater(long owningBucketOrd, GlobalOrdLookupFunction lookupGlobalOrd) throws IOException;
 
         /**
          * Build a {@link PriorityQueue} to sort the buckets. After we've
@@ -675,7 +678,7 @@ public void accept(long globalOrd, long bucketOrd, long docCount) throws IOExcep
         /**
          * Convert a temporary bucket into a real bucket.
          */
-        abstract B convertTempBucketToRealBucket(TB temp) throws IOException;
+        abstract B convertTempBucketToRealBucket(TB temp, GlobalOrdLookupFunction lookupGlobalOrd) throws IOException;
 
         /**
          * Build the sub-aggregations into the buckets. This will usually
@@ -735,7 +738,7 @@ OrdBucket buildEmptyTemporaryBucket() {
         }
 
         @Override
-        BucketUpdater<OrdBucket> bucketUpdater(long owningBucketOrd) throws IOException {
+        BucketUpdater<OrdBucket> bucketUpdater(long owningBucketOrd, GlobalOrdLookupFunction lookupGlobalOrd) throws IOException {
             return (spare, globalOrd, bucketOrd, docCount) -> {
                 spare.globalOrd = globalOrd;
                 spare.bucketOrd = bucketOrd;
@@ -748,7 +751,8 @@ PriorityQueue<OrdBucket> buildPriorityQueue(int size) {
             return new BucketPriorityQueue<>(size, partiallyBuiltBucketComparator);
         }
 
-        StringTerms.Bucket convertTempBucketToRealBucket(OrdBucket temp) throws IOException {
+        @Override
+        StringTerms.Bucket convertTempBucketToRealBucket(OrdBucket temp, GlobalOrdLookupFunction lookupGlobalOrd) throws IOException {
             BytesRef term = BytesRef.deepCopyOf(lookupGlobalOrd.apply(temp.globalOrd));
             StringTerms.Bucket result = new StringTerms.Bucket(term, temp.docCount, null, showTermDocCountError, 0, format);
             result.bucketOrd = temp.bucketOrd;
@@ -871,7 +875,8 @@ private long subsetSize(long owningBucketOrd) {
         }
 
         @Override
-        BucketUpdater<SignificantStringTerms.Bucket> bucketUpdater(long owningBucketOrd) throws IOException {
+        BucketUpdater<SignificantStringTerms.Bucket> bucketUpdater(long owningBucketOrd, GlobalOrdLookupFunction lookupGlobalOrd)
+            throws IOException {
             long subsetSize = subsetSize(owningBucketOrd);
             return (spare, globalOrd, bucketOrd, docCount) -> {
                 spare.bucketOrd = bucketOrd;
@@ -895,7 +900,10 @@ PriorityQueue<SignificantStringTerms.Bucket> buildPriorityQueue(int size) {
         }
 
         @Override
-        SignificantStringTerms.Bucket convertTempBucketToRealBucket(SignificantStringTerms.Bucket temp) throws IOException {
+        SignificantStringTerms.Bucket convertTempBucketToRealBucket(
+            SignificantStringTerms.Bucket temp,
+            GlobalOrdLookupFunction lookupGlobalOrd
+        ) throws IOException {
             return temp;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java
index 9fa9e65eb145a..5311688ceee54 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java
@@ -136,6 +136,11 @@ public boolean supportsSampling() {
         return true;
     }
 
+    @Override
+    public boolean supportsParallelCollection() {
+        return false;
+    }
+
     @Override
     protected ValuesSourceType defaultValueSourceType() {
         return CoreValuesSourceType.KEYWORD;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorFactory.java
index d8a3e9097dc5c..2cadbd3d43494 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorFactory.java
@@ -371,7 +371,7 @@ Aggregator create(
                     factories,
                     a -> a.new SignificantTermsResults(lookup, significanceHeuristic, cardinality),
                     ordinalsValuesSource,
-                    values,
+                    () -> TermsAggregatorFactory.globalOrdsValues(context, ordinalsValuesSource),
                     null,
                     format,
                     bucketCountThresholds,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregatorFromFilters.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregatorFromFilters.java
index 9f5c2b0136155..0e0db3ab5054f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregatorFromFilters.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregatorFromFilters.java
@@ -12,6 +12,7 @@
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PriorityQueue;
+import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.search.DocValueFormat;
@@ -62,12 +63,11 @@ static StringTermsAggregatorFromFilters adaptIntoFiltersOrNull(
         BucketOrder order,
         BucketCountThresholds bucketCountThresholds,
         LongPredicate acceptedOrds,
-        SortedSetDocValues values
+        CheckedSupplier<SortedSetDocValues, IOException> valuesSupplier
     ) throws IOException {
         if (false == valuesSourceConfig.alignesWithSearchIndex()) {
             return null;
         }
-        TermsEnum terms = values.termsEnum();
         FilterByFilterAggregator.AdapterBuilder<StringTermsAggregatorFromFilters> filterByFilterBuilder =
             new FilterByFilterAggregator.AdapterBuilder<StringTermsAggregatorFromFilters>(
                 name,
@@ -91,10 +91,12 @@ protected StringTermsAggregatorFromFilters adapt(
                         valuesSourceConfig.format(),
                         order,
                         bucketCountThresholds,
-                        terms
+                        valuesSupplier
                     );
                 }
             };
+        SortedSetDocValues values = valuesSupplier.get();
+        TermsEnum terms = values.termsEnum();
         String field = valuesSourceConfig.fieldContext().field();
         for (long ord = 0; ord < values.getValueCount(); ord++) {
             if (acceptedOrds.test(ord) == false) {
@@ -119,7 +121,7 @@ protected StringTermsAggregatorFromFilters adapt(
     private final DocValueFormat format;
     private final BucketOrder order;
     private final BucketCountThresholds bucketCountThresholds;
-    private final TermsEnum terms;
+    private final CheckedSupplier<SortedSetDocValues, IOException> valuesSupplier;
 
     public StringTermsAggregatorFromFilters(
         Aggregator parent,
@@ -129,14 +131,14 @@ public StringTermsAggregatorFromFilters(
         DocValueFormat format,
         BucketOrder order,
         BucketCountThresholds bucketCountThresholds,
-        TermsEnum terms
+        CheckedSupplier<SortedSetDocValues, IOException> valuesSupplier
     ) throws IOException {
         super(parent, subAggregators, delegate);
         this.showTermDocCountError = showTermDocCountError;
         this.format = format;
         this.order = order;
         this.bucketCountThresholds = bucketCountThresholds;
-        this.terms = terms;
+        this.valuesSupplier = valuesSupplier;
     }
 
     @Override
@@ -160,6 +162,7 @@ protected InternalAggregation adapt(InternalAggregation delegateResult) throws I
         if (minDocCount == 0 && bucketCountThresholds.getMinDocCount() > 0) {
             minDocCount = 1;
         }
+        TermsEnum terms = valuesSupplier.get().termsEnum();
         if (filters.getBuckets().size() > bucketCountThresholds.getShardSize()) {
             PriorityQueue<OrdBucket> queue = new PriorityQueue<OrdBucket>(bucketCountThresholds.getShardSize()) {
                 private final Comparator<Bucket> comparator = order.comparator();
@@ -190,7 +193,7 @@ protected boolean lessThan(OrdBucket a, OrdBucket b) {
             buckets = new ArrayList<>(queue.size());
             if (isKeyOrder(order) == false) {
                 for (OrdBucket b : queue) {
-                    buckets.add(buildBucket(b));
+                    buckets.add(buildBucket(b, terms));
                 }
                 Collections.sort(buckets, reduceOrder.comparator());
             } else {
@@ -201,7 +204,7 @@ protected boolean lessThan(OrdBucket a, OrdBucket b) {
                  * instead of O(n), but such is life. And n shouldn't be too big.
                  */
                 while (queue.size() > 0) {
-                    buckets.add(buildBucket(queue.pop()));
+                    buckets.add(buildBucket(queue.pop(), terms));
                 }
                 // The buckets come off last to first so we need to flip them.
                 Collections.reverse(buckets);
@@ -212,7 +215,7 @@ protected boolean lessThan(OrdBucket a, OrdBucket b) {
                 if (b.getDocCount() < minDocCount) {
                     continue;
                 }
-                buckets.add(buildBucket(b));
+                buckets.add(buildBucket(b, terms));
             }
             Collections.sort(buckets, reduceOrder.comparator());
         }
@@ -232,12 +235,12 @@ protected boolean lessThan(OrdBucket a, OrdBucket b) {
         );
     }
 
-    private StringTerms.Bucket buildBucket(OrdBucket b) throws IOException {
+    private StringTerms.Bucket buildBucket(OrdBucket b, TermsEnum terms) throws IOException {
         terms.seekExact(b.globalOrd);
         return new StringTerms.Bucket(BytesRef.deepCopyOf(terms.term()), b.getDocCount(), b.aggregations, showTermDocCountError, 0, format);
     }
 
-    private StringTerms.Bucket buildBucket(InternalFilters.InternalBucket b) throws IOException {
+    private StringTerms.Bucket buildBucket(InternalFilters.InternalBucket b, TermsEnum terms) throws IOException {
         terms.seekExact(Long.parseLong(b.getKey()));
         return new StringTerms.Bucket(
             BytesRef.deepCopyOf(terms.term()),
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
index 2bc9820a52e7a..e13ee43df5746 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
@@ -439,7 +439,7 @@ Aggregator create(
                         order,
                         bucketCountThresholds,
                         gloabalOrdsFilter(includeExclude, valuesSourceConfig.format(), values),
-                        values
+                        () -> globalOrdsValues(context, ordinalsValuesSource)
                     );
                     if (adapted != null) {
                         /*
@@ -485,7 +485,7 @@ Aggregator create(
                         factories,
                         a -> a.new StandardTermsResults(),
                         ordinalsValuesSource,
-                        values,
+                        () -> globalOrdsValues(context, ordinalsValuesSource),
                         order,
                         valuesSourceConfig.format(),
                         bucketCountThresholds,
@@ -529,7 +529,7 @@ Aggregator create(
                     factories,
                     a -> a.new StandardTermsResults(),
                     ordinalsValuesSource,
-                    values,
+                    () -> globalOrdsValues(context, ordinalsValuesSource),
                     order,
                     valuesSourceConfig.format(),
                     bucketCountThresholds,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
index ec147507f8970..822dd6d983e5c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
@@ -552,7 +552,9 @@ public void addReleasable(Aggregator aggregator) {
         }
 
         @Override
-        public void removeReleasable(Aggregator aggregator) {
+        public synchronized void removeReleasable(Aggregator aggregator) {
+            // Removing an aggregator is done after calling Aggregator#buildTopLevel which happens on an executor thread.
+            // We need to synchronize the removal because he AggregatorContext it is shared between executor threads.
             assert releaseMe.contains(aggregator)
                 : "removing non-existing aggregator [" + aggregator.name() + "] from the the aggregation context";
             releaseMe.remove(aggregator);
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index 6bf4267c3a42f..94bc5a4187435 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -37,6 +37,7 @@
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.SparseFixedBitSet;
 import org.apache.lucene.util.ThreadInterruptedException;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.lucene.util.CombinedBitSet;
@@ -91,6 +92,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
     // don't create slices with less than this number of docs
     private final int minimumDocsPerSlice;
 
+    private final Set<Thread> timeoutOverwrites = ConcurrentCollections.newConcurrentSet();
     private volatile boolean timeExceeded = false;
 
     /** constructor for non-concurrent search */
@@ -345,6 +347,7 @@ public <C extends Collector, T> T search(Query query, CollectorManager<C, T> col
             weight = createWeight(query, firstCollector.scoreMode(), 1);
         } catch (@SuppressWarnings("unused") TimeExceededException e) {
             timeExceeded = true;
+            doAggregationPostCollection(firstCollector);
             return collectorManager.reduce(Collections.singletonList(firstCollector));
         }
         return search(weight, collectorManager, firstCollector);
@@ -475,14 +478,30 @@ public boolean isCancelled() {
     @Override
     public void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
         collector.setWeight(weight);
+        boolean success = false;
         try {
             for (LeafReaderContext ctx : leaves) { // search each subreader
                 searchLeaf(ctx, weight, collector);
             }
+            success = true;
         } catch (@SuppressWarnings("unused") TimeExceededException e) {
             timeExceeded = true;
         } finally {
-            doAggregationPostCollection(collector);
+            // Only run post collection if we have timeout or if the search was successful
+            // otherwise the state of the aggregation might be undefined and running post collection
+            // might result in an exception
+            if (success || timeExceeded) {
+                try {
+                    // Search phase has finished, no longer need to check for timeout
+                    // otherwise the aggregation post-collection phase might get cancelled.
+                    boolean added = timeoutOverwrites.add(Thread.currentThread());
+                    assert added;
+                    doAggregationPostCollection(collector);
+                } finally {
+                    boolean removed = timeoutOverwrites.remove(Thread.currentThread());
+                    assert removed;
+                }
+            }
         }
     }
 
@@ -498,7 +517,9 @@ public boolean timeExceeded() {
     }
 
     public void throwTimeExceededException() {
-        throw new TimeExceededException();
+        if (timeoutOverwrites.contains(Thread.currentThread()) == false) {
+            throw new TimeExceededException();
+        }
     }
 
     private static class TimeExceededException extends RuntimeException {
diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/InternalProfileCollector.java b/server/src/main/java/org/elasticsearch/search/profile/query/InternalProfileCollector.java
index 6db91d16074ca..65a8463afab73 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/query/InternalProfileCollector.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/query/InternalProfileCollector.java
@@ -63,7 +63,7 @@ protected String deriveCollectorName(Collector c) {
 
         // Aggregation collector toString()'s include the user-defined agg name
         if (getReason().equals(CollectorResult.REASON_AGGREGATION) || getReason().equals(CollectorResult.REASON_AGGREGATION_GLOBAL)) {
-            s += ": [" + c + "]";
+            s += ": " + c;
         }
         return s;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index d7d19059cd56d..3044d15ab8552 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -135,7 +135,6 @@ static void executeQuery(SearchContext searchContext) throws QueryPhaseExecution
 
         RescorePhase.execute(searchContext);
         SuggestPhase.execute(searchContext);
-        AggregationPhase.execute(searchContext);
 
         if (searchContext.getProfilers() != null) {
             searchContext.queryResult().profileResults(searchContext.getProfilers().buildQueryPhaseResults());
@@ -202,36 +201,28 @@ static void addCollectorsAndSearch(SearchContext searchContext) throws QueryPhas
                 searcher.addQueryCancellation(timeoutRunnable);
             }
 
-            try {
-                QueryPhaseResult queryPhaseResult = searcher.search(query, collectorManager);
-                if (searchContext.getProfilers() != null) {
-                    searchContext.getProfilers().getCurrentQueryProfiler().setCollectorResult(queryPhaseResult.collectorResult());
-                }
-                queryResult.topDocs(queryPhaseResult.topDocsAndMaxScore(), queryPhaseResult.sortValueFormats());
-                if (searcher.timeExceeded()) {
-                    assert timeoutRunnable != null : "TimeExceededException thrown even though timeout wasn't set";
-                    if (searchContext.request().allowPartialSearchResults() == false) {
-                        throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded");
-                    }
-                    queryResult.searchTimedOut(true);
-                }
-                if (searchContext.terminateAfter() != SearchContext.DEFAULT_TERMINATE_AFTER) {
-                    queryResult.terminatedEarly(queryPhaseResult.terminatedAfter());
-                }
-                ExecutorService executor = searchContext.indexShard().getThreadPool().executor(ThreadPool.Names.SEARCH);
-                assert executor instanceof TaskExecutionTimeTrackingEsThreadPoolExecutor
-                    || (executor instanceof EsThreadPoolExecutor == false /* in case thread pool is mocked out in tests */)
-                    : "SEARCH threadpool should have an executor that exposes EWMA metrics, but is of type " + executor.getClass();
-                if (executor instanceof TaskExecutionTimeTrackingEsThreadPoolExecutor rExecutor) {
-                    queryResult.nodeQueueSize(rExecutor.getCurrentQueueSize());
-                    queryResult.serviceTimeEWMA((long) rExecutor.getTaskExecutionEWMA());
-                }
-            } finally {
-                // Search phase has finished, no longer need to check for timeout
-                // otherwise aggregation phase might get cancelled.
-                if (timeoutRunnable != null) {
-                    searcher.removeQueryCancellation(timeoutRunnable);
+            QueryPhaseResult queryPhaseResult = searcher.search(query, collectorManager);
+            if (searchContext.getProfilers() != null) {
+                searchContext.getProfilers().getCurrentQueryProfiler().setCollectorResult(queryPhaseResult.collectorResult());
+            }
+            queryResult.topDocs(queryPhaseResult.topDocsAndMaxScore(), queryPhaseResult.sortValueFormats());
+            if (searcher.timeExceeded()) {
+                assert timeoutRunnable != null : "TimeExceededException thrown even though timeout wasn't set";
+                if (searchContext.request().allowPartialSearchResults() == false) {
+                    throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded");
                 }
+                queryResult.searchTimedOut(true);
+            }
+            if (searchContext.terminateAfter() != SearchContext.DEFAULT_TERMINATE_AFTER) {
+                queryResult.terminatedEarly(queryPhaseResult.terminatedAfter());
+            }
+            ExecutorService executor = searchContext.indexShard().getThreadPool().executor(ThreadPool.Names.SEARCH);
+            assert executor instanceof TaskExecutionTimeTrackingEsThreadPoolExecutor
+                || (executor instanceof EsThreadPoolExecutor == false /* in case thread pool is mocked out in tests */)
+                : "SEARCH threadpool should have an executor that exposes EWMA metrics, but is of type " + executor.getClass();
+            if (executor instanceof TaskExecutionTimeTrackingEsThreadPoolExecutor rExecutor) {
+                queryResult.nodeQueueSize(rExecutor.getCurrentQueueSize());
+                queryResult.serviceTimeEWMA((long) rExecutor.getTaskExecutionEWMA());
             }
         } catch (Exception e) {
             throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Failed to execute main query", e);
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollectorManager.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollectorManager.java
index afdfbd785908c..86a01756d247e 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollectorManager.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhaseCollectorManager.java
@@ -50,6 +50,7 @@
 import org.elasticsearch.lucene.grouping.SinglePassGroupingCollector;
 import org.elasticsearch.lucene.grouping.TopFieldGroups;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AggregatorCollector;
 import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
@@ -76,14 +77,14 @@
 abstract class QueryPhaseCollectorManager implements CollectorManager<Collector, QueryPhaseResult> {
     private final Weight postFilterWeight;
     private final QueryPhaseCollector.TerminateAfterChecker terminateAfterChecker;
-    private final CollectorManager<? extends Collector, Void> aggsCollectorManager;
+    private final CollectorManager<AggregatorCollector, Void> aggsCollectorManager;
     private final Float minScore;
     private final boolean profile;
 
     QueryPhaseCollectorManager(
         Weight postFilterWeight,
         QueryPhaseCollector.TerminateAfterChecker terminateAfterChecker,
-        CollectorManager<? extends Collector, Void> aggsCollectorManager,
+        CollectorManager<AggregatorCollector, Void> aggsCollectorManager,
         Float minScore,
         boolean profile
     ) {
@@ -147,7 +148,7 @@ public final QueryPhaseResult reduce(Collection<Collector> collectors) throws IO
         boolean terminatedAfter = false;
         CollectorResult collectorResult = null;
         List<Collector> topDocsCollectors = new ArrayList<>();
-        List<Collector> aggsCollectors = new ArrayList<>();
+        List<AggregatorCollector> aggsCollectors = new ArrayList<>();
         if (profile) {
             List<CollectorResult> resultsPerProfiler = new ArrayList<>();
             List<CollectorResult> topDocsCollectorResults = new ArrayList<>();
@@ -165,7 +166,7 @@ public final QueryPhaseResult reduce(Collection<Collector> collectors) throws IO
                 if (aggsCollectorManager != null) {
                     InternalProfileCollector profileAggsCollector = (InternalProfileCollector) queryPhaseCollector.getAggsCollector();
                     aggsCollectorResults.add(profileAggsCollector.getCollectorTree());
-                    aggsCollectors.add(profileAggsCollector.getWrappedCollector());
+                    aggsCollectors.add((AggregatorCollector) profileAggsCollector.getWrappedCollector());
                 }
             }
             List<CollectorResult> childrenResults = new ArrayList<>();
@@ -178,16 +179,14 @@ public final QueryPhaseResult reduce(Collection<Collector> collectors) throws IO
             for (Collector collector : collectors) {
                 QueryPhaseCollector queryPhaseCollector = (QueryPhaseCollector) collector;
                 topDocsCollectors.add(queryPhaseCollector.getTopDocsCollector());
-                aggsCollectors.add(queryPhaseCollector.getAggsCollector());
+                aggsCollectors.add((AggregatorCollector) queryPhaseCollector.getAggsCollector());
                 if (queryPhaseCollector.isTerminatedAfter()) {
                     terminatedAfter = true;
                 }
             }
         }
         if (aggsCollectorManager != null) {
-            @SuppressWarnings("unchecked")
-            CollectorManager<Collector, Void> aggsManager = (CollectorManager<Collector, Void>) aggsCollectorManager;
-            aggsManager.reduce(aggsCollectors);
+            aggsCollectorManager.reduce(aggsCollectors);
         }
         TopDocsAndMaxScore topDocsAndMaxScore = reduceTopDocsCollectors(topDocsCollectors);
         return new QueryPhaseResult(topDocsAndMaxScore, getSortValueFormats(), terminatedAfter, collectorResult);
@@ -212,7 +211,7 @@ private static CollectorResult reduceCollectorResults(Collection<CollectorResult
      */
     static CollectorManager<Collector, QueryPhaseResult> createQueryPhaseCollectorManager(
         Weight postFilterWeight,
-        CollectorManager<? extends Collector, Void> aggsCollectorManager,
+        CollectorManager<AggregatorCollector, Void> aggsCollectorManager,
         SearchContext searchContext,
         boolean hasFilterCollector
     ) throws IOException {
@@ -310,7 +309,7 @@ private static final class EmptyHits extends QueryPhaseCollectorManager {
         EmptyHits(
             Weight postFilterWeight,
             QueryPhaseCollector.TerminateAfterChecker terminateAfterChecker,
-            CollectorManager<? extends Collector, Void> aggsCollectorManager,
+            CollectorManager<AggregatorCollector, Void> aggsCollectorManager,
             Float minScore,
             boolean profile,
             @Nullable SortAndFormats sortAndFormats,
@@ -375,7 +374,7 @@ private static class WithHits extends QueryPhaseCollectorManager {
         WithHits(
             Weight postFilterWeight,
             QueryPhaseCollector.TerminateAfterChecker terminateAfterChecker,
-            CollectorManager<? extends Collector, Void> aggsCollectorManager,
+            CollectorManager<AggregatorCollector, Void> aggsCollectorManager,
             Float minScore,
             boolean profile,
             IndexReader reader,
@@ -480,7 +479,7 @@ protected final DocValueFormat[] getSortValueFormats() {
     private static WithHits forScroll(
         Weight postFilterWeight,
         QueryPhaseCollector.TerminateAfterChecker terminateAfterChecker,
-        CollectorManager<? extends Collector, Void> aggsCollectorManager,
+        CollectorManager<AggregatorCollector, Void> aggsCollectorManager,
         Float minScore,
         boolean profile,
         IndexReader reader,
@@ -540,7 +539,7 @@ public TopDocsAndMaxScore reduceTopDocsCollectors(Collection<Collector> collecto
     private static QueryPhaseCollectorManager forCollapsing(
         Weight postFilterWeight,
         QueryPhaseCollector.TerminateAfterChecker terminateAfterChecker,
-        CollectorManager<? extends Collector, Void> aggsCollectorManager,
+        CollectorManager<AggregatorCollector, Void> aggsCollectorManager,
         Float minScore,
         boolean profile,
         CollapseContext collapseContext,
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index ee7a8723a5a27..d19fd4ecf08a3 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -24,7 +24,6 @@
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.sandbox.document.HalfFloatPoint;
 import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.CollectorManager;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
@@ -122,7 +121,6 @@
 import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
 import org.elasticsearch.search.aggregations.metrics.MultiValueAggregation;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.AggregationContext.ProductionAggregationContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
@@ -150,7 +148,6 @@
 import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -161,6 +158,7 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
+import java.util.function.Supplier;
 import java.util.stream.Stream;
 
 import static java.util.Collections.emptyMap;
@@ -294,19 +292,6 @@ protected AggregationContext createAggregationContext(IndexReader indexReader, Q
             AggregationBuilder.DEFAULT_PREALLOCATION * 5, // We don't know how many bytes to preallocate so we grab a hand full
             DEFAULT_MAX_BUCKETS,
             false,
-            fieldTypes
-        );
-    }
-
-    private AggregationContext createAggregationContext(IndexSearcher indexSearcher, Query query, MappedFieldType... fieldTypes)
-        throws IOException {
-        return createAggregationContext(
-            indexSearcher,
-            createIndexSettings(),
-            query,
-            new NoneCircuitBreakerService(),
-            AggregationBuilder.DEFAULT_PREALLOCATION * 5, // We don't know how many bytes to preallocate so we grab a hand full
-            DEFAULT_MAX_BUCKETS,
             false,
             fieldTypes
         );
@@ -329,10 +314,11 @@ protected AggregationContext createAggregationContext(
         long bytesToPreallocate,
         int maxBucket,
         boolean isInSortOrderExecutionRequired,
+        boolean supportsParallelCollection,
         MappedFieldType... fieldTypes
     ) throws IOException {
         return createAggregationContext(
-            newIndexSearcher(indexReader),
+            newIndexSearcher(indexReader, supportsParallelCollection),
             indexSettings,
             query,
             breakerService,
@@ -510,7 +496,7 @@ protected ScriptService getMockScriptService() {
      */
     protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduce(IndexReader reader, AggTestConfig aggTestConfig)
         throws IOException {
-        IndexSearcher searcher = newIndexSearcher(reader);
+        IndexSearcher searcher = newIndexSearcher(reader, aggTestConfig.builder.supportsParallelCollection());
         IndexSettings indexSettings = createIndexSettings();
         // First run it to find circuit breaker leaks on the aggregator
         runWithCrankyCircuitBreaker(indexSettings, searcher, aggTestConfig);
@@ -553,9 +539,9 @@ private <A extends InternalAggregation, C extends Aggregator> A searchAndReduce(
         MappedFieldType[] fieldTypes = aggTestConfig.fieldTypes();
 
         final IndexReaderContext ctx = searcher.getTopReaderContext();
-        final PipelineTree pipelines = builder.buildPipelineTree();
-        List<InternalAggregation> internalAggs = new ArrayList<>();
+        List<InternalAggregations> internalAggs = new ArrayList<>();
         Query rewritten = searcher.rewrite(query);
+        BigArrays bigArraysForReduction = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), breakerService);
 
         if (splitLeavesIntoSeparateAggregators
             && searcher.getIndexReader().leaves().size() > 0
@@ -590,7 +576,9 @@ private <A extends InternalAggregation, C extends Aggregator> A searchAndReduce(
                         a.postCollection();
                     }
                     assertEquals(shouldBeCached, context.isCacheable());
-                    internalAggs.add(a.buildTopLevel());
+                    List<InternalAggregation> internalAggregations = List.of(a.buildTopLevel());
+                    assertRoundTrip(internalAggregations);
+                    internalAggs.add(InternalAggregations.from(internalAggregations));
                 } finally {
                     Releasables.close(context);
                 }
@@ -613,54 +601,56 @@ private <A extends InternalAggregation, C extends Aggregator> A searchAndReduce(
                     root.preCollection();
                     aggregators.add(root);
                     new TimeSeriesIndexSearcher(searcher, List.of()).search(rewritten, MultiBucketCollector.wrap(true, List.of(root)));
+                    List<InternalAggregation> internalAggregations = List.of(root.buildTopLevel());
+                    assertRoundTrip(internalAggregations);
+                    internalAggs.add(InternalAggregations.from(internalAggregations));
                 } else {
-                    CollectorManager<Collector, Void> collectorManager = new CollectorManager<>() {
-                        @Override
-                        public Collector newCollector() throws IOException {
-                            C collector = createAggregator(builder, context);
-                            collector.preCollection();
-                            aggregators.add(collector);
-                            return MultiBucketCollector.wrap(true, List.of(collector)).asCollector();
-                        }
-
-                        @Override
-                        public Void reduce(Collection<Collector> collectors) {
-                            return null;
+                    Supplier<AggregatorCollector> aggregatorSupplier = () -> {
+                        try {
+                            Aggregator aggregator = createAggregator(builder, context);
+                            aggregator.preCollection();
+                            BucketCollector bucketCollector = MultiBucketCollector.wrap(true, List.of(aggregator));
+                            return new AggregatorCollector(new Aggregator[] { aggregator }, bucketCollector);
+                        } catch (IOException e) {
+                            throw new AggregationInitializationException("Could not initialize aggregators", e);
                         }
                     };
-                    if (aggTestConfig.builder().supportsParallelCollection()) {
-                        searcher.search(rewritten, collectorManager);
-                    } else {
-                        searcher.search(rewritten, collectorManager.newCollector());
-                    }
-                }
-                for (C agg : aggregators) {
-                    internalAggs.add(agg.buildTopLevel());
+                    Supplier<AggregationReduceContext> reduceContextSupplier = () -> new AggregationReduceContext.ForPartial(
+                        bigArraysForReduction,
+                        getMockScriptService(),
+                        () -> false,
+                        builder
+                    );
+                    AggregatorCollectorManager aggregatorCollectorManager = new AggregatorCollectorManager(
+                        aggregatorSupplier,
+                        internalAggs::add,
+                        reduceContextSupplier
+                    );
+                    searcher.search(rewritten, aggregatorCollectorManager);
                 }
             } finally {
                 Releasables.close(context);
             }
         }
-        assertRoundTrip(internalAggs);
 
-        BigArrays bigArraysForReduction = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), breakerService);
         try {
             if (aggTestConfig.incrementalReduce() && internalAggs.size() > 1) {
                 // sometimes do an incremental reduce
                 int toReduceSize = internalAggs.size();
                 Collections.shuffle(internalAggs, random());
                 int r = randomIntBetween(1, toReduceSize);
-                List<InternalAggregation> toReduce = internalAggs.subList(0, r);
+                List<InternalAggregations> toReduce = internalAggs.subList(0, r);
                 AggregationReduceContext reduceContext = new AggregationReduceContext.ForPartial(
                     bigArraysForReduction,
                     getMockScriptService(),
                     () -> false,
                     builder
                 );
-                A reduced = (A) doReduce(toReduce, reduceContext);
                 internalAggs = new ArrayList<>(internalAggs.subList(r, toReduceSize));
-                internalAggs.add(reduced);
-                assertRoundTrip(internalAggs);
+                internalAggs.add(InternalAggregations.topLevelReduce(toReduce, reduceContext));
+                for (InternalAggregations internalAggregation : internalAggs) {
+                    assertRoundTrip(internalAggregation.copyResults());
+                }
             }
 
             // now do the final reduce
@@ -677,12 +667,9 @@ public Void reduce(Collection<Collector> collectors) {
             );
 
             @SuppressWarnings("unchecked")
-            A internalAgg = (A) doReduce(internalAggs, reduceContext);
+            A internalAgg = (A) doInternalAggregationsReduce(internalAggs, reduceContext);
             assertRoundTrip(internalAgg);
 
-            // materialize any parent pipelines
-            internalAgg = (A) internalAgg.reducePipelines(internalAgg, reduceContext, pipelines);
-
             doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer);
             assertRoundTrip(internalAgg);
             if (aggTestConfig.builder instanceof ValuesSourceAggregationBuilder.MetricsAggregationBuilder<?>) {
@@ -699,6 +686,13 @@ private InternalAggregation doReduce(List<InternalAggregation> aggregators, Aggr
         for (InternalAggregation aggregator : aggregators) {
             internalAggregations.add(InternalAggregations.from(List.of(aggregator)));
         }
+        return doInternalAggregationsReduce(internalAggregations, reduceContext);
+    }
+
+    private InternalAggregation doInternalAggregationsReduce(
+        List<InternalAggregations> internalAggregations,
+        AggregationReduceContext reduceContext
+    ) {
         InternalAggregations aggregations = InternalAggregations.topLevelReduce(internalAggregations, reduceContext);
         List<InternalAggregation> reduced = aggregations.copyResults();
         assertThat(reduced.size(), equalTo(1));
@@ -807,7 +801,7 @@ protected <R extends InternalAggregation> void debugTestCase(
         MappedFieldType... fieldTypes
     ) throws IOException {
         // Don't use searchAndReduce because we only want a single aggregator.
-        IndexSearcher searcher = newIndexSearcher(reader);
+        IndexSearcher searcher = newIndexSearcher(reader, aggregationBuilder.supportsParallelCollection());
         if (queryCachingPolicy != null) {
             searcher.setQueryCachingPolicy(queryCachingPolicy);
         }
@@ -955,7 +949,7 @@ protected static DirectoryReader wrapInMockESDirectoryReader(DirectoryReader dir
     /**
      * Creates a {@link ContextIndexSearcher} that supports concurrency running each segment in a different thread.
      */
-    private IndexSearcher newIndexSearcher(IndexReader indexReader) throws IOException {
+    private IndexSearcher newIndexSearcher(IndexReader indexReader, boolean supportsParallelCollection) throws IOException {
         return new ContextIndexSearcher(
             indexReader,
             IndexSearcher.getDefaultSimilarity(),
@@ -964,7 +958,7 @@ private IndexSearcher newIndexSearcher(IndexReader indexReader) throws IOExcepti
             indexReader instanceof DirectoryReader ? randomBoolean() : false, // we can only wrap DirectoryReader instances
             this.threadPoolExecutor,
             this.threadPoolExecutor.getMaximumPoolSize(),
-            1 // forces multiple slices
+            supportsParallelCollection ? 1 /* forces multiple slices */ : Integer.MAX_VALUE // one slice
         );
     }
 
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
index 9abd4ca5faa59..cd36af992b0ab 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
@@ -378,6 +378,7 @@ public void testTonsOfBucketsTriggersBreaker() throws IOException {
                         breaker,
                         builder.bytesToPreallocate(),
                         MultiBucketConsumerService.DEFAULT_MAX_BUCKETS,
+                        true,
                         false,
                         doubleFields()
                     )
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSource.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSource.java
index 24e8da132ca46..8a7d2afa958d9 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSource.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSource.java
@@ -183,7 +183,7 @@ static class GlobalOrdinalsStrategy implements ExecutionStrategy {
 
             private final Field field;
             private final Bytes.WithOrdinals source;
-            private final SortedSetDocValues docValues;
+            private SortedSetDocValues docValues;
             private final LongBitSet bitSetFilter;
 
             GlobalOrdinalsStrategy(
@@ -198,15 +198,15 @@ static class GlobalOrdinalsStrategy implements ExecutionStrategy {
                 bitSetFilter = globalOrdinalsFilter != null
                     ? globalOrdinalsFilter.acceptedGlobalOrdinals(source.globalOrdinalsValues(ctx))
                     : null;
-
                 this.docValues = source.globalOrdinalsValues(ctx);
             }
 
             @Override
             public ValueCollector getValueCollector(LeafReaderContext ctx) throws IOException {
-                final SortedSetDocValues values = source.globalOrdinalsValues(ctx);
+                this.docValues = source.globalOrdinalsValues(ctx);
+                ;
                 final Tuple<Field, List<Object>> empty = new Tuple<>(field, Collections.emptyList());
-
+                final SortedSetDocValues values = this.docValues;
                 return doc -> {
                     if (values.advanceExact(doc)) {
                         int valuesCount = values.docValueCount();

From 53f2eefa366352b043f936ee9404e4b50b36013c Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 19 Sep 2023 09:38:10 +0100
Subject: [PATCH 34/41] Expand Javadocs on writeIndexGen (#99645)

Adds a little more context describing why we use the three-step process,
and a note about the meaning of a failure at this step.
---
 .../cluster/metadata/RepositoryMetadata.java  | 17 +++++++---
 .../blobstore/BlobStoreRepository.java        | 34 +++++++++++++++----
 2 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java
index 5218598083103..95bb9d1431f3d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java
@@ -106,8 +106,13 @@ public Settings settings() {
     /**
      * Returns the safe repository generation. {@link RepositoryData} for this generation is assumed to exist in the repository.
      * All operations on the repository must be based on the {@link RepositoryData} at this generation.
-     * See package level documentation for the blob store based repositories {@link org.elasticsearch.repositories.blobstore} for details
-     * on how this value is used during snapshots.
+     * <p>
+     * This value is always at most {@link #pendingGeneration()}, and will be equal to {@link #pendingGeneration()} iff there are no ongoing
+     * root-blob writes.
+     * <p>
+     * See the package level documentation for {@link org.elasticsearch.repositories.blobstore} for details on how this value is used during
+     * snapshots.
+
      * @return safe repository generation
      */
     public long generation() {
@@ -118,8 +123,12 @@ public long generation() {
      * Returns the pending repository generation. {@link RepositoryData} for this generation and all generations down to the safe
      * generation {@link #generation} may exist in the repository and should not be reused for writing new {@link RepositoryData} to the
      * repository.
-     * See package level documentation for the blob store based repositories {@link org.elasticsearch.repositories.blobstore} for details
-     * on how this value is used during snapshots.
+     * <p>
+     * This value is always at least {@link #generation()}, and will be equal to {@link #generation()} iff there are no ongoing root-blob
+     * writes.
+     * <p>
+     * See the package level documentation for {@link org.elasticsearch.repositories.blobstore} for details on how this value is used during
+     * snapshots.
      *
      * @return highest pending repository generation
      */
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 436d34bc39c21..c0ef6581db94b 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -2191,12 +2191,34 @@ public boolean isReadOnly() {
     }
 
     /**
-     * Writing a new index generation is a three step process.
-     * First, the {@link RepositoryMetadata} entry for this repository is set into a pending state by incrementing its
-     * pending generation {@code P} while its safe generation {@code N} remains unchanged.
-     * Second, the updated {@link RepositoryData} is written to generation {@code P + 1}.
-     * Lastly, the {@link RepositoryMetadata} entry for this repository is updated to the new generation {@code P + 1} and thus
-     * pending and safe generation are set to the same value marking the end of the update of the repository data.
+     * Writing a new index generation (root) blob is a three-step process. Typically, it starts from a stable state where the pending
+     * generation {@link RepositoryMetadata#pendingGeneration()} is equal to the safe generation {@link RepositoryMetadata#generation()},
+     * but after a failure it may be that the pending generation starts out greater than the safe generation.
+     * <ol>
+     * <li>
+     * We reserve ourselves a new root blob generation {@code G}, greater than {@link RepositoryMetadata#pendingGeneration()}, via a
+     * cluster state update which edits the {@link RepositoryMetadata} entry for this repository, increasing its pending generation to
+     * {@code G} without changing its safe generation.
+     * <li>
+     * We write the updated {@link RepositoryData} to a new root blob with generation {@code G}.
+     * <li>
+     * We mark the successful end of the update of the repository data with a cluster state update which edits the
+     * {@link RepositoryMetadata} entry for this repository again, increasing its safe generation to equal to its pending generation
+     * {@code G}.
+     * </ol>
+     * We use this process to protect against problems such as a master failover part-way through. If a new master is elected while we're
+     * writing the root blob with generation {@code G} then we will fail to update the safe repository generation in the final step, and
+     * meanwhile the new master will choose a generation greater than {@code G} for all subsequent root blobs so there is no risk that we
+     * will clobber its writes. See the package level documentation for {@link org.elasticsearch.repositories.blobstore} for more details.
+     * <p>
+     * Note that a failure here does not imply that the process was unsuccessful or the repository is unchanged. Once we have written the
+     * new root blob the repository is updated from the point of view of any other clusters reading from it, and if we performed a full
+     * cluster restart at that point then we would also pick up the new root blob. Writing the root blob may succeed without us receiving
+     * a successful response from the repository, leading us to report that the write operation failed. Updating the safe generation may
+     * likewise succeed on a majority of master-eligible nodes which does not include this one, again leading to an apparent failure.
+     * <p>
+     * We therefore cannot safely clean up apparently-dangling blobs after a failure here. Instead, we defer any cleanup until after the
+     * next successful root-blob write, which may happen on a different master node or possibly even in a different cluster.
      *
      * @param repositoryData RepositoryData to write
      * @param expectedGen    expected repository generation at the start of the operation

From f04af725fe0d00d99ba7e455eb608f1bc018e7c6 Mon Sep 17 00:00:00 2001
From: Andrei Stefan <astefan@users.noreply.github.com>
Date: Tue, 19 Sep 2023 11:55:50 +0300
Subject: [PATCH 35/41] ESQL: continue resolving attributes for Eval (#99601)

---
 docs/changelog/99601.yaml                     |  6 ++++
 .../src/main/resources/eval.csv-spec          | 33 +++++++++++++++++++
 .../xpack/esql/analysis/Analyzer.java         | 12 ++++---
 .../xpack/esql/analysis/AnalyzerTests.java    | 19 +++++++++++
 4 files changed, 66 insertions(+), 4 deletions(-)
 create mode 100644 docs/changelog/99601.yaml

diff --git a/docs/changelog/99601.yaml b/docs/changelog/99601.yaml
new file mode 100644
index 0000000000000..9deba859a5cef
--- /dev/null
+++ b/docs/changelog/99601.yaml
@@ -0,0 +1,6 @@
+pr: 99601
+summary: "ESQL: continue resolving attributes for Eval"
+area: ES|QL
+type: bug
+issues:
+ - 99576
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec
index 9108b0d6eedb6..45d1967973eb0 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec
@@ -166,3 +166,36 @@ emp_no:i | sum:i
 10002 | 3230647
 10003 | 3230970
 ; 
+
+chainedEvalReusingPreviousValue
+from employees | sort emp_no | eval x1 = concat(first_name, "."), x2 = concat(x1, "."), x3 = concat(x2, ".") | keep x*, first_name | limit 5;
+
+   x1:keyword  |   x2:keyword  |  x3:keyword   |first_name:keyword   
+Georgi.        |Georgi..       |Georgi...      |Georgi         
+Bezalel.       |Bezalel..      |Bezalel...     |Bezalel        
+Parto.         |Parto..        |Parto...       |Parto          
+Chirstian.     |Chirstian..    |Chirstian...   |Chirstian      
+Kyoichi.       |Kyoichi..      |Kyoichi...     |Kyoichi
+;
+
+chainedEvalReusingPreviousValue2
+from employees | sort emp_no | eval x1 = concat(first_name, "."), x2 = concat(x1, last_name), x3 = concat(x2, gender) | keep x*, first_name, gender | limit 5;
+
+   x1:keyword  |   x2:keyword    |  x3:keyword      |first_name:keyword|gender:keyword
+Georgi.        |Georgi.Facello   |Georgi.FacelloM   |Georgi            |M              
+Bezalel.       |Bezalel.Simmel   |Bezalel.SimmelF   |Bezalel           |F              
+Parto.         |Parto.Bamford    |Parto.BamfordM    |Parto             |M              
+Chirstian.     |Chirstian.Koblick|Chirstian.KoblickM|Chirstian         |M              
+Kyoichi.       |Kyoichi.Maliniak |Kyoichi.MaliniakM |Kyoichi           |M
+;
+
+chainedEvalReusingPreviousValue3
+from employees | sort emp_no | eval x1 = concat(first_name, "."), x2 = concat(x1, last_name), x3 = concat(x2, x1) | keep x*, first_name | limit 5;
+
+   x1:keyword  |   x2:keyword    |        x3:keyword         |first_name:keyword
+Georgi.        |Georgi.Facello   |Georgi.FacelloGeorgi.      |Georgi         
+Bezalel.       |Bezalel.Simmel   |Bezalel.SimmelBezalel.     |Bezalel        
+Parto.         |Parto.Bamford    |Parto.BamfordParto.        |Parto          
+Chirstian.     |Chirstian.Koblick|Chirstian.KoblickChirstian.|Chirstian      
+Kyoichi.       |Kyoichi.Maliniak |Kyoichi.MaliniakKyoichi.   |Kyoichi
+;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
index 5d7217c7d1ded..e511874afa486 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
@@ -323,13 +323,17 @@ protected LogicalPlan doRule(LogicalPlan plan) {
                 return resolveEnrich(p, childrenOutput);
             }
 
-            return plan.transformExpressionsUp(UnresolvedAttribute.class, ua -> resolveAttribute(ua, childrenOutput));
+            return plan.transformExpressionsUp(UnresolvedAttribute.class, ua -> maybeResolveAttribute(ua, childrenOutput));
         }
 
-        private Attribute resolveAttribute(UnresolvedAttribute ua, List<Attribute> childrenOutput) {
+        private Attribute maybeResolveAttribute(UnresolvedAttribute ua, List<Attribute> childrenOutput) {
             if (ua.customMessage()) {
                 return ua;
             }
+            return resolveAttribute(ua, childrenOutput);
+        }
+
+        private Attribute resolveAttribute(UnresolvedAttribute ua, List<Attribute> childrenOutput) {
             Attribute resolved = ua;
             var named = resolveAgainstList(ua, childrenOutput);
             // if resolved, return it; otherwise keep it in place to be resolved later
@@ -441,7 +445,7 @@ private LogicalPlan resolveRename(Rename rename, List<Attribute> childrenOutput)
                     // remove attributes overwritten by a renaming: `| keep a, b, c | rename a as b`
                     projections.removeIf(x -> x.name().equals(alias.name()));
 
-                    var resolved = resolveAttribute(ua, childrenOutput);
+                    var resolved = maybeResolveAttribute(ua, childrenOutput);
                     if (resolved instanceof UnsupportedAttribute || resolved.resolved()) {
                         var realiased = (NamedExpression) alias.replaceChildren(List.of(resolved));
                         projections.replaceAll(x -> x.equals(resolved) ? realiased : x);
@@ -490,7 +494,7 @@ private LogicalPlan resolveRename(Rename rename, List<Attribute> childrenOutput)
         private LogicalPlan resolveEnrich(Enrich enrich, List<Attribute> childrenOutput) {
 
             if (enrich.matchField().toAttribute() instanceof UnresolvedAttribute ua) {
-                Attribute resolved = resolveAttribute(ua, childrenOutput);
+                Attribute resolved = maybeResolveAttribute(ua, childrenOutput);
                 if (resolved.equals(ua)) {
                     return enrich;
                 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index 06cbddac631c7..6e480749efb21 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -33,6 +33,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.stream.IntStream;
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze;
@@ -1254,6 +1255,24 @@ public void testEnrichExcludesPolicyKey() {
         assertThat(e.getMessage(), containsString("Unknown column [id]"));
     }
 
+    public void testChainedEvalFieldsUse() {
+        var query = "from test | eval x0 = pow(salary, 1), x1 = pow(x0, 2), x2 = pow(x1, 3)";
+        int additionalEvals = randomIntBetween(0, 5);
+        for (int i = 0, j = 3; i < additionalEvals; i++, j++) {
+            query += ", x" + j + " = pow(x" + (j - 1) + ", " + i + ")";
+        }
+        assertProjection(query + " | keep x*", IntStream.range(0, additionalEvals + 3).mapToObj(v -> "x" + v).toArray(String[]::new));
+    }
+
+    public void testMissingAttributeException_InChainedEval() {
+        var e = expectThrows(VerificationException.class, () -> analyze("""
+            from test
+            | eval x1 = concat(first_name, "."), x2 = concat(x1, last_name), x3 = concat(x2, x1), x4 = concat(x3, x5)
+            | keep x*
+            """));
+        assertThat(e.getMessage(), containsString("Unknown column [x5], did you mean any of [x1, x2, x3]?"));
+    }
+
     private void verifyUnsupported(String query, String errorMessage) {
         verifyUnsupported(query, errorMessage, "mapping-multi-field-variation.json");
     }

From 4fdfb3354284f0a604f86de9e5a21bd7cebc8618 Mon Sep 17 00:00:00 2001
From: Ignacio Vera <ivera@apache.org>
Date: Tue, 19 Sep 2023 11:07:26 +0200
Subject: [PATCH 36/41] mute BreakingBytesRefBuilderTests#testGrow (#99651)

relates https://github.com/elastic/elasticsearch/issues/99649
---
 .../compute/operator/BreakingBytesRefBuilderTests.java           | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java
index 21e74e5a22fcf..08eb57f85a908 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java
@@ -72,6 +72,7 @@ public void applyToOracle(BytesRefBuilder oracle) {
         });
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99649")
     public void testGrow() {
         testAgainstOracle(() -> new TestIteration() {
             int length = between(1, 100);

From f685bdca9b695f6a5f91fe8d7f77b4147de99d16 Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Tue, 19 Sep 2023 12:46:45 +0300
Subject: [PATCH 37/41] [TEST] `illegal_argument_exception` when downsampling
 all indexes in serverless (#99630)

Downsampling wildcard and _all indexes raises different errors in
stateful (security disabled) vs serverless (security enabled):

Stateful:

```
[{type=index_not_found_exception, reason=no such index [test*], index_uuid=_na_, index=test*, stack_trace=[test*] org.elasticsearch.index.IndexNotFoundException: no such index [test*]\n\tat org.elasticsearch.xpack.downsample.TransportDownsampleAction.masterOperation(TransportDownsampleAction.java:205)
```

Serverless:

```
[{type=illegal_argument_exception, reason=the action indices:admin/xpack/downsample does not support wildcards; the provided index expression(s) [test*] are not allowed, stack_trace=org.elasticsearch.ElasticsearchException$1: the action indices:admin/xpack/downsample does not support wildcards; the provided index expression(s) [test*] are not allowed
```
---
 .../resources/rest-api-spec/test/downsample/10_basic.yml    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml
index f4dd187b82445..70f66f38d39b9 100644
--- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml
+++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml
@@ -444,8 +444,10 @@ setup:
         body:
           index.blocks.write: true
 
+  # The following two tests may return different exceptions, depending on whether security is enabled (serverless)
+  # or not (stateful).
   - do:
-      catch: /index_not_found_exception/
+      catch: /index_not_found_exception|illegal_argument_exception/
       indices.downsample:
         index: test*
         target_index: test-downsample
@@ -455,7 +457,7 @@ setup:
           }
 
   - do:
-      catch: /index_not_found_exception/
+      catch: /index_not_found_exception|illegal_argument_exception/
       indices.downsample:
         index: _all
         target_index: test-downsample

From bf8429b59623889d533f39a5b10f686397953229 Mon Sep 17 00:00:00 2001
From: Luigi Dell'Aquila <luigi.dellaquila@gmail.com>
Date: Tue, 19 Sep 2023 12:04:30 +0200
Subject: [PATCH 38/41] Mute BreakingBytesRefBuilderTests#testAddBytesRef
 (#99653)

Flaky tests

relates https://github.com/elastic/elasticsearch/issues/99649
---
 .../compute/operator/BreakingBytesRefBuilderTests.java           | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java
index 08eb57f85a908..3f03823815249 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java
@@ -51,6 +51,7 @@ public void applyToOracle(BytesRefBuilder oracle) {
         });
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99649")
     public void testAddBytesRef() {
         testAgainstOracle(() -> new TestIteration() {
             BytesRef ref = new BytesRef(randomAlphaOfLengthBetween(1, 100));

From 61562f64b92869046e14482365339428fef67b9a Mon Sep 17 00:00:00 2001
From: Ignacio Vera <ivera@apache.org>
Date: Tue, 19 Sep 2023 12:52:25 +0200
Subject: [PATCH 39/41] Remove supportsParallelCollection implementation for
 some aggs (#99654)

enable Cardinality, composite, nested and Frequent imters aggregations for concurrent execution.
---
 .../bucket/composite/CompositeAggregationBuilder.java       | 5 -----
 .../histogram/VariableWidthHistogramAggregationBuilder.java | 5 -----
 .../bucket/nested/NestedAggregationBuilder.java             | 5 -----
 .../aggregations/metrics/CardinalityAggregationBuilder.java | 5 -----
 .../search/aggregations/AggregatorFactoriesTests.java       | 6 +++---
 .../FrequentItemSetsAggregationBuilder.java                 | 5 -----
 6 files changed, 3 insertions(+), 28 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java
index 6b894b3a9dbbd..ca88d50898763 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java
@@ -97,11 +97,6 @@ public boolean supportsSampling() {
         return true;
     }
 
-    @Override
-    public boolean supportsParallelCollection() {
-        return false;
-    }
-
     public CompositeAggregationBuilder(StreamInput in) throws IOException {
         super(in);
         int num = in.readVInt();
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java
index 9ca606f460fbb..f7870df45648e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java
@@ -137,11 +137,6 @@ public BucketCardinality bucketCardinality() {
         return BucketCardinality.MANY;
     }
 
-    @Override
-    public boolean supportsParallelCollection() {
-        return false;
-    }
-
     @Override
     protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map<String, Object> metaData) {
         return new VariableWidthHistogramAggregationBuilder(this, factoriesBuilder, metaData);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
index b6e6a39356d46..f4ba3db383586 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
@@ -81,11 +81,6 @@ public BucketCardinality bucketCardinality() {
         return BucketCardinality.ONE;
     }
 
-    @Override
-    public boolean supportsParallelCollection() {
-        return false;
-    }
-
     @Override
     protected AggregatorFactory doBuild(AggregationContext context, AggregatorFactory parent, Builder subFactoriesBuilder)
         throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java
index 03e3dc93675d3..615b46434226d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java
@@ -113,11 +113,6 @@ public boolean supportsSampling() {
         return true;
     }
 
-    @Override
-    public boolean supportsParallelCollection() {
-        return false;
-    }
-
     @Override
     protected boolean serializeTargetValueType(TransportVersion version) {
         return true;
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
index bf0b087f67a5a..d67e99e4521f1 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
@@ -334,19 +334,19 @@ public void testSupportsParallelCollection() {
         {
             AggregatorFactories.Builder builder = new AggregatorFactories.Builder();
             builder.addAggregator(new CardinalityAggregationBuilder("cardinality"));
-            assertFalse(builder.supportsParallelCollection());
+            assertTrue(builder.supportsParallelCollection());
         }
         {
             AggregatorFactories.Builder builder = new AggregatorFactories.Builder();
             builder.addAggregator(new NestedAggregationBuilder("nested", "path"));
-            assertFalse(builder.supportsParallelCollection());
+            assertTrue(builder.supportsParallelCollection());
         }
         {
             AggregatorFactories.Builder builder = new AggregatorFactories.Builder();
             builder.addAggregator(
                 new CompositeAggregationBuilder("composite", Collections.singletonList(new TermsValuesSourceBuilder("name")))
             );
-            assertFalse(builder.supportsParallelCollection());
+            assertTrue(builder.supportsParallelCollection());
         }
         {
             AggregatorFactories.Builder builder = new AggregatorFactories.Builder();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilder.java
index 9081f4118f89f..684d61dbdedb3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilder.java
@@ -182,11 +182,6 @@ public boolean supportsSampling() {
         return true;
     }
 
-    @Override
-    public boolean supportsParallelCollection() {
-        return false;
-    }
-
     @Override
     protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map<String, Object> metadata) {
         return new FrequentItemSetsAggregationBuilder(name, fields, minimumSupport, minimumSetSize, size, filter, executionHint);

From c663c831b7e334f980667686d1b0555a605f4bdf Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 19 Sep 2023 12:46:20 +0100
Subject: [PATCH 40/41] Improve async chaining in
 TransportStartDataFrameAnalyticsAction (#99623)

In #99619 we introduced `SubscribableListener#andThen` allowing to chain
async actions together in the order they will be executed. Today there's
several spots in the ML codebase that declare a fairly long sequence of
actions in reverse order, making it hard to read. This commit cleans up
one such place using `SubscribableListener#andThen` to demonstrate its
value.
---
 ...ransportStartDataFrameAnalyticsAction.java | 126 ++++++++----------
 1 file changed, 59 insertions(+), 67 deletions(-)

diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
index 20fa9ba694758..e24ffbff06f98 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
@@ -263,79 +264,70 @@ private void estimateMemoryUsageAndUpdateMemoryTracker(StartContext startContext
     private void getStartContext(String id, Task task, ActionListener<StartContext> finalListener) {
 
         ParentTaskAssigningClient parentTaskClient = new ParentTaskAssigningClient(client, task.getParentTaskId());
-        // Step 7. Validate that there are analyzable data in the source index
-        ActionListener<StartContext> validateMappingsMergeListener = ActionListener.wrap(
-            startContext -> validateSourceIndexHasAnalyzableData(startContext, finalListener),
-            finalListener::onFailure
-        );
 
-        // Step 6. Validate mappings can be merged
-        ActionListener<StartContext> toValidateMappingsListener = ActionListener.wrap(
-            startContext -> MappingsMerger.mergeMappings(
-                parentTaskClient,
-                startContext.config.getHeaders(),
-                startContext.config.getSource(),
-                ActionListener.wrap(mappings -> validateMappingsMergeListener.onResponse(startContext), finalListener::onFailure)
-            ),
-            finalListener::onFailure
-        );
+        SubscribableListener
 
-        // Step 5. Validate dest index is empty if task is starting for first time
-        ActionListener<StartContext> toValidateDestEmptyListener = ActionListener.wrap(startContext -> {
-            switch (startContext.startingState) {
-                case FIRST_TIME -> checkDestIndexIsEmptyIfExists(parentTaskClient, startContext, toValidateMappingsListener);
-                case RESUMING_REINDEXING, RESUMING_ANALYZING, RESUMING_INFERENCE -> toValidateMappingsListener.onResponse(startContext);
-                case FINISHED -> {
-                    logger.info("[{}] Job has already finished", startContext.config.getId());
-                    finalListener.onFailure(ExceptionsHelper.badRequestException("Cannot start because the job has already finished"));
-                }
-                default -> finalListener.onFailure(
-                    ExceptionsHelper.serverError("Unexpected starting state {}", startContext.startingState)
+            // Step 1. Get the config
+            .<DataFrameAnalyticsConfig>newForked(l -> configProvider.get(id, l))
+
+            // Step 2. Get stats to recover progress
+            .<StartContext>andThen((l, config) -> getProgress(config, l.map(progress -> new StartContext(config, progress))))
+
+            // Step 3. Validate source and dest
+            .<StartContext>andThen((l, startContext) -> {
+                // Validate the query parses
+                startContext.config.getSource().getParsedQuery();
+
+                // Validate source/dest are valid
+                sourceDestValidator.validate(
+                    clusterService.state(),
+                    startContext.config.getSource().getIndex(),
+                    startContext.config.getDest().getIndex(),
+                    null,
+                    SourceDestValidations.ALL_VALIDATIONS,
+                    l.map(ignored -> startContext)
                 );
-            }
-        }, finalListener::onFailure);
-
-        // Step 4. Check data extraction is possible
-        ActionListener<StartContext> toValidateExtractionPossibleListener = ActionListener.wrap(startContext -> {
-            new ExtractedFieldsDetectorFactory(parentTaskClient).createFromSource(
-                startContext.config,
-                ActionListener.wrap(extractedFieldsDetector -> {
-                    startContext.extractedFields = extractedFieldsDetector.detect().v1();
-                    toValidateDestEmptyListener.onResponse(startContext);
-                }, finalListener::onFailure)
-            );
-        }, finalListener::onFailure);
-
-        // Step 3. Validate source and dest
-        ActionListener<StartContext> startContextListener = ActionListener.wrap(startContext -> {
-            // Validate the query parses
-            startContext.config.getSource().getParsedQuery();
-
-            // Validate source/dest are valid
-            sourceDestValidator.validate(
-                clusterService.state(),
-                startContext.config.getSource().getIndex(),
-                startContext.config.getDest().getIndex(),
-                null,
-                SourceDestValidations.ALL_VALIDATIONS,
-                ActionListener.wrap(aBoolean -> toValidateExtractionPossibleListener.onResponse(startContext), finalListener::onFailure)
-            );
-        }, finalListener::onFailure);
+            })
 
-        // Step 2. Get stats to recover progress
-        ActionListener<DataFrameAnalyticsConfig> getConfigListener = ActionListener.wrap(
-            config -> getProgress(
-                config,
-                ActionListener.wrap(
-                    progress -> startContextListener.onResponse(new StartContext(config, progress)),
-                    finalListener::onFailure
+            // Step 4. Check data extraction is possible
+            .<StartContext>andThen(
+                (l, startContext) -> new ExtractedFieldsDetectorFactory(parentTaskClient).createFromSource(
+                    startContext.config,
+                    l.map(extractedFieldsDetector -> {
+                        startContext.extractedFields = extractedFieldsDetector.detect().v1();
+                        return startContext;
+                    })
                 )
-            ),
-            finalListener::onFailure
-        );
+            )
+
+            // Step 5. Validate dest index is empty if task is starting for first time
+            .<StartContext>andThen((l, startContext) -> {
+                switch (startContext.startingState) {
+                    case FIRST_TIME -> checkDestIndexIsEmptyIfExists(parentTaskClient, startContext, l);
+                    case RESUMING_REINDEXING, RESUMING_ANALYZING, RESUMING_INFERENCE -> l.onResponse(startContext);
+                    case FINISHED -> {
+                        logger.info("[{}] Job has already finished", startContext.config.getId());
+                        l.onFailure(ExceptionsHelper.badRequestException("Cannot start because the job has already finished"));
+                    }
+                    default -> l.onFailure(ExceptionsHelper.serverError("Unexpected starting state {}", startContext.startingState));
+                }
+            })
+
+            // Step 6. Validate mappings can be merged
+            .<StartContext>andThen(
+                (l, startContext) -> MappingsMerger.mergeMappings(
+                    parentTaskClient,
+                    startContext.config.getHeaders(),
+                    startContext.config.getSource(),
+                    l.map(ignored -> startContext)
+                )
+            )
+
+            // Step 7. Validate that there are analyzable data in the source index
+            .<StartContext>andThen((l, startContext) -> validateSourceIndexHasAnalyzableData(startContext, l))
 
-        // Step 1. Get the config
-        configProvider.get(id, getConfigListener);
+            // Step 8. Respond
+            .addListener(finalListener);
     }
 
     private void validateSourceIndexHasAnalyzableData(StartContext startContext, ActionListener<StartContext> listener) {

From 7f4b9bbe8a4c07338f9edf2e054e37a696c4ece1 Mon Sep 17 00:00:00 2001
From: Quentin Pradet <quentin.pradet@elastic.co>
Date: Tue, 19 Sep 2023 16:28:43 +0400
Subject: [PATCH 41/41] [DOCS] Stop linking to indices.templates.html page
 (#99618)

---
 .../rest-api-spec/api/indices.delete_index_template.json        | 2 +-
 .../resources/rest-api-spec/api/indices.delete_template.json    | 2 +-
 .../rest-api-spec/api/indices.exists_index_template.json        | 2 +-
 .../resources/rest-api-spec/api/indices.exists_template.json    | 2 +-
 .../resources/rest-api-spec/api/indices.get_index_template.json | 2 +-
 .../main/resources/rest-api-spec/api/indices.get_template.json  | 2 +-
 .../resources/rest-api-spec/api/indices.put_index_template.json | 2 +-
 .../main/resources/rest-api-spec/api/indices.put_template.json  | 2 +-
 .../rest-api-spec/api/indices.simulate_index_template.json      | 2 +-
 .../resources/rest-api-spec/api/indices.simulate_template.json  | 2 +-
 10 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_index_template.json
index a07d654078cfc..e920ec09351ec 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_index_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_index_template.json
@@ -1,7 +1,7 @@
 {
   "indices.delete_index_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-delete-template.html",
       "description":"Deletes an index template."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
index 7d79c2aba13c3..b0471b9dafa7e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
@@ -1,7 +1,7 @@
 {
   "indices.delete_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-delete-template-v1.html",
       "description":"Deletes an index template."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_index_template.json
index dc0f9ed59c665..0163b0bbbd8ce 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_index_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_index_template.json
@@ -1,7 +1,7 @@
 {
   "indices.exists_index_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/index-templates.html",
       "description":"Returns information about whether a particular index template exists."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
index 9d2b6b15cc054..77cb62b022515 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
@@ -1,7 +1,7 @@
 {
   "indices.exists_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-template-exists-v1.html",
       "description":"Returns information about whether a particular index template exists."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json
index 7b24fec14c4a0..4ed82c3bbc5eb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json
@@ -1,7 +1,7 @@
 {
   "indices.get_index_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-template.html",
       "description":"Returns an index template."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
index e1f7c1c0202a5..963d9a4cb670e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
@@ -1,7 +1,7 @@
 {
   "indices.get_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-template-v1.html",
       "description":"Returns an index template."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_index_template.json
index 542f316ad7ab8..00f6ac88eab45 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_index_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_index_template.json
@@ -1,7 +1,7 @@
 {
   "indices.put_index_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-put-template.html",
       "description":"Creates or updates an index template."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
index e87c625d1cfb1..7e4afb14e06e6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
@@ -1,7 +1,7 @@
 {
   "indices.put_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates-v1.html",
       "description":"Creates or updates an index template."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json
index 152fae82f070a..e12b5116ffcdf 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json
@@ -1,7 +1,7 @@
 {
   "indices.simulate_index_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-simulate-index.html",
       "description": "Simulate matching the given index name against the index templates in the system"
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_template.json
index 763e46e48fef2..7958454564c83 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_template.json
@@ -1,7 +1,7 @@
 {
   "indices.simulate_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-simulate-template.html",
       "description": "Simulate resolving the given template name or body"
     },
     "stability":"stable",