From a400e3c06d758899f832a3dc391f5921c34ae9ab Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Thu, 21 Mar 2024 14:32:49 -0700
Subject: [PATCH 01/31] Revert "Avoid using small inactive exchange timeout in
 breaker tests (#106394)"

This reverts commit bce11c6c1c1c73a4b9a57b367cc6d379b9acac1b.
---
 .../compute/operator/exchange/ExchangeService.java          | 6 +-----
 .../xpack/esql/action/EsqlActionBreakerIT.java              | 4 +++-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java
index 9af08346256f7..a8afce1a3b223 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java
@@ -25,9 +25,7 @@
 import org.elasticsearch.compute.data.BlockStreamInput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportChannel;
@@ -180,15 +178,13 @@ public void messageReceived(OpenExchangeRequest request, TransportChannel channe
 
     private class ExchangeTransportAction implements TransportRequestHandler<ExchangeRequest> {
         @Override
-        public void messageReceived(ExchangeRequest request, TransportChannel channel, Task transportTask) {
+        public void messageReceived(ExchangeRequest request, TransportChannel channel, Task task) {
             final String exchangeId = request.exchangeId();
             ActionListener<ExchangeResponse> listener = new ChannelActionListener<>(channel);
             final ExchangeSinkHandler sinkHandler = sinks.get(exchangeId);
             if (sinkHandler == null) {
                 listener.onResponse(new ExchangeResponse(blockFactory, null, true));
             } else {
-                CancellableTask task = (CancellableTask) transportTask;
-                task.addListener(() -> sinkHandler.onFailure(new TaskCancelledException(task.getReasonCancelled())));
                 sinkHandler.fetchPageAsync(request.sourcesFinished(), listener);
             }
         }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java
index 059ed672e56c6..85eb0c02625ad 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.action;
 
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.DocWriteResponse;
@@ -34,6 +35,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105543")
 @TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug")
 public class EsqlActionBreakerIT extends EsqlActionIT {
 
@@ -70,7 +72,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
                 HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_TYPE_SETTING.getKey(),
                 HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_TYPE_SETTING.getDefault(Settings.EMPTY)
             )
-            .put(ExchangeService.INACTIVE_SINKS_INTERVAL_SETTING, TimeValue.timeValueSeconds(between(5, 10)))
+            .put(ExchangeService.INACTIVE_SINKS_INTERVAL_SETTING, TimeValue.timeValueMillis(between(500, 2000)))
             .put(BlockFactory.LOCAL_BREAKER_OVER_RESERVED_SIZE_SETTING, ByteSizeValue.ofBytes(between(0, 256)))
             .put(BlockFactory.LOCAL_BREAKER_OVER_RESERVED_MAX_SIZE_SETTING, ByteSizeValue.ofBytes(between(0, 1024)))
             // allow reading pages from network can trip the circuit breaker

From 2b67444a465ca50a6854c36025fd7f23aac5a117 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 21 Mar 2024 15:02:26 -0700
Subject: [PATCH 02/31] Wait indefintely for http connections on shutdown by
 default (#106511)

This commit changes the meaning of the http shutdown timeout when set to
0. Previously the shutdown would proceed immediately, killing active
http connections. With this change 0 means no timeout is used, but
active connections are still waited on, indefinitely. This new behavior
is what was originally intended for the default value of the shutdown
timeout setting.
---
 docs/changelog/106511.yaml                    |  5 ++
 .../common/settings/ClusterSettings.java      |  1 +
 .../http/AbstractHttpServerTransport.java     | 34 ++++++++--
 .../http/HttpTransportSettings.java           |  6 ++
 .../AbstractHttpServerTransportTests.java     | 63 +++++++++++++++----
 5 files changed, 91 insertions(+), 18 deletions(-)
 create mode 100644 docs/changelog/106511.yaml

diff --git a/docs/changelog/106511.yaml b/docs/changelog/106511.yaml
new file mode 100644
index 0000000000000..bdef7f1aea225
--- /dev/null
+++ b/docs/changelog/106511.yaml
@@ -0,0 +1,5 @@
+pr: 106511
+summary: Wait indefintely for http connections on shutdown by default
+area: Infra/Node Lifecycle
+type: bug
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 21801dee844b0..ac5255f58622a 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -313,6 +313,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE,
         HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH,
         HttpTransportSettings.SETTING_HTTP_SERVER_SHUTDOWN_GRACE_PERIOD,
+        HttpTransportSettings.SETTING_HTTP_SERVER_SHUTDOWN_POLL_PERIOD,
         HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT,
         HttpTransportSettings.SETTING_HTTP_RESET_COOKIES,
         HttpTransportSettings.SETTING_HTTP_TCP_NO_DELAY,
diff --git a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
index f9005f6e37889..3f95aff62313b 100644
--- a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
+++ b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
@@ -70,6 +70,7 @@
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_SERVER_SHUTDOWN_GRACE_PERIOD;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_SERVER_SHUTDOWN_POLL_PERIOD;
 
 public abstract class AbstractHttpServerTransport extends AbstractLifecycleComponent implements HttpServerTransport {
     private static final Logger logger = LogManager.getLogger(AbstractHttpServerTransport.class);
@@ -95,6 +96,7 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo
     private final RefCounted refCounted = AbstractRefCounted.of(() -> allClientsClosedListener.onResponse(null));
     private final Set<HttpServerChannel> httpServerChannels = ConcurrentCollections.newConcurrentSet();
     private final long shutdownGracePeriodMillis;
+    private final long shutdownPollPeriodMillis;
     private final HttpClientStatsTracker httpClientStatsTracker;
 
     private final HttpTracer httpLogger;
@@ -146,6 +148,7 @@ protected AbstractHttpServerTransport(
         slowLogThresholdMs = TransportSettings.SLOW_OPERATION_THRESHOLD_SETTING.get(settings).getMillis();
         httpClientStatsTracker = new HttpClientStatsTracker(settings, clusterSettings, threadPool);
         shutdownGracePeriodMillis = SETTING_HTTP_SERVER_SHUTDOWN_GRACE_PERIOD.get(settings).getMillis();
+        shutdownPollPeriodMillis = SETTING_HTTP_SERVER_SHUTDOWN_POLL_PERIOD.get(settings).getMillis();
     }
 
     public Recycler<BytesRef> recycler() {
@@ -272,17 +275,36 @@ protected void doStop() {
 
         boolean closed = false;
 
+        long pollTimeMillis = shutdownPollPeriodMillis;
         if (shutdownGracePeriodMillis > 0) {
+            if (shutdownGracePeriodMillis < pollTimeMillis) {
+                pollTimeMillis = shutdownGracePeriodMillis;
+            }
+            logger.debug(format("waiting [%d]ms for clients to close connections", shutdownGracePeriodMillis));
+        } else {
+            logger.debug("waiting indefinitely for clients to close connections");
+        }
+
+        long startPollTimeMillis = System.currentTimeMillis();
+        do {
             try {
-                logger.debug(format("waiting [%d]ms for clients to close connections", shutdownGracePeriodMillis));
-                FutureUtils.get(allClientsClosedListener, shutdownGracePeriodMillis, TimeUnit.MILLISECONDS);
+                FutureUtils.get(allClientsClosedListener, pollTimeMillis, TimeUnit.MILLISECONDS);
                 closed = true;
             } catch (ElasticsearchTimeoutException t) {
-                logger.warn(format("timed out while waiting [%d]ms for clients to close connections", shutdownGracePeriodMillis));
+                logger.info(format("still waiting on %d client connections to close", httpChannels.size()));
+                if (shutdownGracePeriodMillis > 0) {
+                    long endPollTimeMillis = System.currentTimeMillis();
+                    long remainingGracePeriodMillis = shutdownGracePeriodMillis - (endPollTimeMillis - startPollTimeMillis);
+                    if (remainingGracePeriodMillis <= 0) {
+                        logger.warn(format("timed out while waiting [%d]ms for clients to close connections", shutdownGracePeriodMillis));
+                        break;
+                    } else if (remainingGracePeriodMillis < pollTimeMillis) {
+                        pollTimeMillis = remainingGracePeriodMillis;
+                    }
+                }
             }
-        } else {
-            logger.debug("closing all client connections immediately");
-        }
+        } while (closed == false);
+
         if (closed == false) {
             try {
                 CloseableChannel.closeChannels(new ArrayList<>(httpChannels.values()), true);
diff --git a/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
index 737a99d536919..dcceb43b63db8 100644
--- a/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
+++ b/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
@@ -132,6 +132,12 @@ public final class HttpTransportSettings {
         Setting.Property.NodeScope
     );
 
+    public static final Setting<TimeValue> SETTING_HTTP_SERVER_SHUTDOWN_POLL_PERIOD = Setting.positiveTimeSetting(
+        "http.shutdown_poll_period",
+        TimeValue.timeValueMinutes(5),
+        Setting.Property.NodeScope
+    );
+
     // don't reset cookies by default, since I don't think we really need to
     // note, parsing cookies was fixed in netty 3.5.1 regarding stack allocation, but still, currently, we don't need cookies
     public static final Setting<Boolean> SETTING_HTTP_RESET_COOKIES = Setting.boolSetting("http.reset_cookies", false, Property.NodeScope);
diff --git a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java
index 7f5c623dbae08..06d05f9dc06fa 100644
--- a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java
+++ b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java
@@ -918,8 +918,8 @@ protected void stopInternal() {}
         }
     }
 
-    public void testStopDoesntWaitIfGraceIsZero() {
-        try (var noWait = LogExpectation.unexpectWait(); var transport = new TestHttpServerTransport(Settings.EMPTY)) {
+    public void testStopWaitsIndefinitelyIfGraceIsZero() {
+        try (var wait = LogExpectation.expectWait(); var transport = new TestHttpServerTransport(Settings.EMPTY)) {
             TestHttpChannel httpChannel = new TestHttpChannel();
             transport.serverAcceptedChannel(httpChannel);
             transport.incomingRequest(testHttpRequest(), httpChannel);
@@ -927,7 +927,33 @@ public void testStopDoesntWaitIfGraceIsZero() {
             transport.doStop();
             assertFalse(transport.testHttpServerChannel.isOpen());
             assertFalse(httpChannel.isOpen());
-            noWait.assertExpectationsMatched();
+            wait.assertExpectationsMatched();
+        }
+    }
+
+    public void testStopLogsProgress() throws Exception {
+        TestHttpChannel httpChannel = new TestHttpChannel();
+        var doneWithRequest = new CountDownLatch(1);
+        try (var wait = LogExpectation.expectUpdate(1); var transport = new TestHttpServerTransport(gracePeriod(SHORT_GRACE_PERIOD_MS))) {
+
+            httpChannel.blockSendResponse();
+            var inResponse = httpChannel.notifyInSendResponse();
+
+            transport.serverAcceptedChannel(httpChannel);
+            new Thread(() -> {
+                transport.incomingRequest(testHttpRequest(), httpChannel);
+                doneWithRequest.countDown();
+            }, "testStopLogsProgress -> incomingRequest").start();
+
+            inResponse.await();
+
+            transport.doStop();
+            assertFalse(transport.testHttpServerChannel.isOpen());
+            assertFalse(httpChannel.isOpen());
+            wait.assertExpectationsMatched();
+        } finally {
+            httpChannel.allowSendResponse();
+            doneWithRequest.await();
         }
     }
 
@@ -1345,20 +1371,24 @@ private LogExpectation(int grace) {
         }
 
         public static LogExpectation expectTimeout(int grace) {
-            return new LogExpectation(grace).timedOut(true).wait(true);
+            return new LogExpectation(grace).timedOut(true).wait(false);
         }
 
         public static LogExpectation unexpectedTimeout(int grace) {
-            return new LogExpectation(grace).timedOut(false).wait(true);
+            return new LogExpectation(grace).timedOut(false).wait(false);
+        }
+
+        public static LogExpectation expectWait() {
+            return new LogExpectation(0).wait(true);
         }
 
-        public static LogExpectation unexpectWait() {
-            return new LogExpectation(0).wait(false);
+        public static LogExpectation expectUpdate(int connections) {
+            return new LogExpectation(0).update(connections);
         }
 
         private LogExpectation timedOut(boolean expected) {
             var message = "timed out while waiting [" + grace + "]ms for clients to close connections";
-            var name = "message";
+            var name = "timed out message";
             var logger = AbstractHttpServerTransport.class.getName();
             var level = Level.WARN;
             if (expected) {
@@ -1370,18 +1400,27 @@ private LogExpectation timedOut(boolean expected) {
         }
 
         private LogExpectation wait(boolean expected) {
-            var message = "closing all client connections immediately";
-            var name = "message";
+            var message = "waiting indefinitely for clients to close connections";
+            var name = "wait message";
             var logger = AbstractHttpServerTransport.class.getName();
             var level = Level.DEBUG;
             if (expected) {
-                appender.addExpectation(new MockLogAppender.UnseenEventExpectation(name, logger, level, message));
-            } else {
                 appender.addExpectation(new MockLogAppender.SeenEventExpectation(name, logger, level, message));
+            } else {
+                appender.addExpectation(new MockLogAppender.UnseenEventExpectation(name, logger, level, message));
             }
             return this;
         }
 
+        private LogExpectation update(int connections) {
+            var message = "still waiting on " + connections + " client connections to close";
+            var name = "update message";
+            var logger = AbstractHttpServerTransport.class.getName();
+            var level = Level.INFO;
+            appender.addExpectation(new MockLogAppender.SeenEventExpectation(name, logger, level, message));
+            return this;
+        }
+
         public void assertExpectationsMatched() {
             appender.assertAllExpectationsMatched();
             checked = true;

From ff91ce024c31cacc6ea420a5a5e74a35b610b512 Mon Sep 17 00:00:00 2001
From: Athena Brown <athena.brown@elastic.co>
Date: Thu, 21 Mar 2024 17:17:25 -0600
Subject: [PATCH 03/31] Update and pin gradle wrapper validation action
 (#106635)

This commit updates the Gradle wrapper validation action to the latest
release and pins it to that version by hash per our company best
practices for GitHub Workflows & Actions.
---
 .github/workflows/gradle-wrapper-validation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/gradle-wrapper-validation.yml b/.github/workflows/gradle-wrapper-validation.yml
index 39bef5e91f954..46762905e2c9a 100644
--- a/.github/workflows/gradle-wrapper-validation.yml
+++ b/.github/workflows/gradle-wrapper-validation.yml
@@ -10,4 +10,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: gradle/wrapper-validation-action@v1
+      - uses: gradle/wrapper-validation-action@699bb18358f12c5b78b37bb0111d3a0e2276e0e2 # Release v2.1.1

From a960befd28cb17a21bd5c671c8a1b5d7d4fb656d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= <lorenzo.dematte@elastic.co>
Date: Fri, 22 Mar 2024 00:52:44 +0100
Subject: [PATCH 04/31] Adding `resolveConcreteIndexNames` to
 DataStreamsActionUtil (#106621)

Extract method `resolveConcreteIndexNames` from
`DataStreamsStatsTransportAction` to `DataStreamsActionUtil`, and move
the whole class to the `server` module, so it can be shared and used
across plugins.
---
 .../DataStreamsStatsTransportAction.java      |  25 +---
 .../DeleteDataStreamTransportAction.java      |   2 +-
 .../action/GetDataStreamsTransportAction.java |   1 +
 ...nsportDeleteDataStreamLifecycleAction.java |   2 +-
 ...TransportGetDataStreamLifecycleAction.java |   2 +-
 ...TransportPutDataStreamLifecycleAction.java |   2 +-
 .../datastreams}/DataStreamsActionUtil.java   |  29 ++++-
 .../DataStreamsActionUtilTests.java           | 107 ++++++++++++++++++
 8 files changed, 146 insertions(+), 24 deletions(-)
 rename {modules/data-streams/src/main/java/org/elasticsearch/datastreams/action => server/src/main/java/org/elasticsearch/action/datastreams}/DataStreamsActionUtil.java (58%)
 create mode 100644 server/src/test/java/org/elasticsearch/action/datastreams/DataStreamsActionUtilTests.java

diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
index 2aa5c07ad6be5..c7d70fa06162d 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.PointValues;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.datastreams.DataStreamsActionUtil;
 import org.elasticsearch.action.datastreams.DataStreamsStatsAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
@@ -44,7 +45,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedMap;
-import java.util.stream.Stream;
 
 public class DataStreamsStatsTransportAction extends TransportBroadcastByNodeAction<
     DataStreamsStatsAction.Request,
@@ -104,25 +104,12 @@ protected ClusterBlockException checkRequestBlock(
 
     @Override
     protected String[] resolveConcreteIndexNames(ClusterState clusterState, DataStreamsStatsAction.Request request) {
-        List<String> abstractionNames = indexNameExpressionResolver.dataStreamNames(
+        return DataStreamsActionUtil.resolveConcreteIndexNames(
+            indexNameExpressionResolver,
             clusterState,
-            request.indicesOptions(),
-            request.indices()
-        );
-        SortedMap<String, IndexAbstraction> indicesLookup = clusterState.getMetadata().getIndicesLookup();
-
-        String[] concreteDatastreamIndices = abstractionNames.stream().flatMap(abstractionName -> {
-            IndexAbstraction indexAbstraction = indicesLookup.get(abstractionName);
-            assert indexAbstraction != null;
-            if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
-                DataStream dataStream = (DataStream) indexAbstraction;
-                List<Index> indices = dataStream.getIndices();
-                return indices.stream().map(Index::getName);
-            } else {
-                return Stream.empty();
-            }
-        }).toArray(String[]::new);
-        return concreteDatastreamIndices;
+            request.indices(),
+            request.indicesOptions()
+        ).toArray(String[]::new);
     }
 
     @Override
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java
index 6e7528c470d49..c3e8331b856fd 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java
@@ -44,7 +44,7 @@
 import java.util.Set;
 import java.util.function.Consumer;
 
-import static org.elasticsearch.datastreams.action.DataStreamsActionUtil.getDataStreamNames;
+import static org.elasticsearch.action.datastreams.DataStreamsActionUtil.getDataStreamNames;
 
 public class DeleteDataStreamTransportAction extends AcknowledgedTransportMasterNodeAction<DeleteDataStreamAction.Request> {
 
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java
index 2b1d4ae01f565..41e62508cafbb 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java
@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.datastreams.DataStreamsActionUtil;
 import org.elasticsearch.action.datastreams.GetDataStreamAction;
 import org.elasticsearch.action.datastreams.GetDataStreamAction.Response.IndexProperties;
 import org.elasticsearch.action.datastreams.GetDataStreamAction.Response.ManagedBy;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java
index 0381014aed24b..9683588bdcae3 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportDeleteDataStreamLifecycleAction.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.datastreams.lifecycle.action;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.datastreams.DataStreamsActionUtil;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction;
@@ -19,7 +20,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.datastreams.action.DataStreamsActionUtil;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java
index 84144cdcb0379..3a3a54d747920 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleAction.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.datastreams.lifecycle.action;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.datastreams.DataStreamsActionUtil;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.cluster.ClusterState;
@@ -21,7 +22,6 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.datastreams.action.DataStreamsActionUtil;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java
index 31d7237eeb681..7a33d3011c621 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportPutDataStreamLifecycleAction.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.datastreams.lifecycle.action;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.datastreams.DataStreamsActionUtil;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction;
@@ -19,7 +20,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.datastreams.action.DataStreamsActionUtil;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsActionUtil.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java
similarity index 58%
rename from modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsActionUtil.java
rename to server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java
index a05de46d0e12a..e9ff74459c153 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsActionUtil.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java
@@ -6,13 +6,18 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.datastreams.action;
+package org.elasticsearch.action.datastreams;
 
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.index.Index;
 
 import java.util.List;
+import java.util.SortedMap;
+import java.util.stream.Stream;
 
 public class DataStreamsActionUtil {
 
@@ -40,4 +45,26 @@ public static IndicesOptions updateIndicesOptions(IndicesOptions indicesOptions)
         }
         return indicesOptions;
     }
+
+    public static Stream<String> resolveConcreteIndexNames(
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        ClusterState clusterState,
+        String[] names,
+        IndicesOptions indicesOptions
+    ) {
+        List<String> abstractionNames = getDataStreamNames(indexNameExpressionResolver, clusterState, names, indicesOptions);
+        SortedMap<String, IndexAbstraction> indicesLookup = clusterState.getMetadata().getIndicesLookup();
+
+        return abstractionNames.stream().flatMap(abstractionName -> {
+            IndexAbstraction indexAbstraction = indicesLookup.get(abstractionName);
+            assert indexAbstraction != null;
+            if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
+                DataStream dataStream = (DataStream) indexAbstraction;
+                List<Index> indices = dataStream.getIndices();
+                return indices.stream().map(Index::getName);
+            } else {
+                return Stream.empty();
+            }
+        });
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/DataStreamsActionUtilTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/DataStreamsActionUtilTests.java
new file mode 100644
index 0000000000000..81c8abed352ab
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/datastreams/DataStreamsActionUtilTests.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.datastreams;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamMetadata;
+import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DataStreamsActionUtilTests extends ESTestCase {
+
+    public void testDataStreamsResolveConcreteIndexNames() {
+
+        var index1 = new Index("foo1", IndexMetadata.INDEX_UUID_NA_VALUE);
+        var index3 = new Index("bar", IndexMetadata.INDEX_UUID_NA_VALUE);
+
+        var dataStreamIndex1 = new Index(".ds-foo1", IndexMetadata.INDEX_UUID_NA_VALUE);
+        var dataStreamIndex2 = new Index(".ds-bar2", IndexMetadata.INDEX_UUID_NA_VALUE);
+        var dataStreamIndex3 = new Index(".ds-foo2", IndexMetadata.INDEX_UUID_NA_VALUE);
+        var dataStreamIndex4 = new Index(".ds-baz1", IndexMetadata.INDEX_UUID_NA_VALUE);
+
+        ClusterState clusterState = ClusterState.builder(new ClusterName("test-cluster"))
+            .metadata(
+                Metadata.builder()
+                    .putCustom(
+                        DataStreamMetadata.TYPE,
+                        new DataStreamMetadata(
+                            ImmutableOpenMap.<String, DataStream>builder()
+                                .fPut("fooDs", DataStreamTestHelper.newInstance("fooDs", List.of(dataStreamIndex1)))
+                                .fPut("barDs", DataStreamTestHelper.newInstance("barDs", List.of(dataStreamIndex2)))
+                                .fPut("foo2Ds", DataStreamTestHelper.newInstance("foo2Ds", List.of(dataStreamIndex3)))
+                                .fPut("bazDs", DataStreamTestHelper.newInstance("bazDs", List.of(dataStreamIndex4)))
+                                .build(),
+                            ImmutableOpenMap.of()
+                        )
+                    )
+                    .indices(
+                        createLocalOnlyIndicesMetadata(
+                            index1,
+                            index3,
+                            dataStreamIndex1,
+                            dataStreamIndex2,
+                            dataStreamIndex3,
+                            dataStreamIndex4
+                        )
+                    )
+                    .build()
+            )
+            .build();
+
+        var query = new String[] { "foo*", "baz*" };
+        var indexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
+        when(indexNameExpressionResolver.dataStreamNames(any(), any(), eq(query))).thenReturn(List.of("fooDs", "foo2Ds", "bazDs"));
+
+        var resolved = DataStreamsActionUtil.resolveConcreteIndexNames(
+            indexNameExpressionResolver,
+            clusterState,
+            query,
+            IndicesOptions.builder().wildcardOptions(IndicesOptions.WildcardOptions.builder().includeHidden(true)).build()
+        ).toList();
+
+        assertThat(resolved, containsInAnyOrder(".ds-foo1", ".ds-foo2", ".ds-baz1"));
+    }
+
+    private Map<String, IndexMetadata> createLocalOnlyIndicesMetadata(Index... indices) {
+        return Arrays.stream(indices)
+            .map(
+                index1 -> Map.entry(
+                    index1.getName(),
+                    IndexMetadata.builder(index1.getName())
+                        .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
+                        .numberOfReplicas(0)
+                        .numberOfShards(1)
+                        .build()
+                )
+            )
+            .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));
+    }
+
+}

From 12e64a1bddac15b49655b79e73347256e371e4fe Mon Sep 17 00:00:00 2001
From: Moritz Mack <mmack@apache.org>
Date: Fri, 22 Mar 2024 09:11:30 +0100
Subject: [PATCH 05/31] Use historical features for Yaml REST tests for cluster
 apis (#106587)

---
 .../test/cat.indices/10_basic.yml             |  4 +-
 .../test/cat.templates/10_basic.yml           | 16 ++--
 .../cluster.allocation_explain/10_basic.yml   | 10 +--
 .../cluster.component_template/10_basic.yml   | 16 ++--
 .../test/cluster.desired_balance/10_basic.yml | 36 ++++-----
 .../test/cluster.desired_nodes/20_dry_run.yml | 10 +--
 .../test/cluster.health/10_basic.yml          | 11 +--
 .../cluster.health/30_indices_options.yml     |  4 +-
 .../test/cluster.info/10_info_all.yml         |  4 +-
 .../test/cluster.info/20_info_http.yml        |  4 +-
 .../test/cluster.info/30_info_thread_pool.yml |  4 +-
 .../test/cluster.info/40_info_script.yml      |  4 +-
 .../10_basic.yml                              | 18 ++---
 .../test/cluster.stats/10_basic.yml           | 24 +++---
 .../cluster.stats/20_indexing_pressure.yml    |  4 +-
 .../rest/yaml/YamlTestLegacyFeatures.java     | 79 +++++++++++++++++--
 16 files changed, 158 insertions(+), 90 deletions(-)

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/10_basic.yml
index 2d006f3425790..981a934a719ca 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.indices/10_basic.yml
@@ -72,7 +72,7 @@
 "Test cat indices output for closed index (pre 7.2.0)":
   - skip:
       reason:  "closed indices are replicated starting version 7.2.0"
-      cluster_features: ["cat_indices_replicate_closed"]
+      cluster_features: ["indices_replicate_closed"]
   - requires:
       test_runner_features: ["allowed_warnings"]
 
@@ -117,7 +117,7 @@
 "Test cat indices output for closed index":
   - skip:
       reason:  "closed indices are replicated starting version 7.2.0"
-      cluster_features: ["cat_indices_replicate_closed"]
+      cluster_features: ["indices_replicate_closed"]
   - requires:
       test_runner_features: ["allowed_warnings"]
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/10_basic.yml
index f6f20913e402b..5270d215f8cea 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.templates/10_basic.yml
@@ -1,7 +1,7 @@
 ---
 "Help":
     - requires:
-        cluster_features: ["cat_templates_v2"]
+        cluster_features: ["templates_v2"]
         reason: "format changed in 7.8 to accomodate V2 index templates"
 
     - do:
@@ -32,7 +32,7 @@
 ---
 "Normal templates":
     - requires:
-        cluster_features: [ "cat_templates_v2" ]
+        cluster_features: [ "templates_v2" ]
         reason: "format changed in 7.8 to accomodate V2 index templates"
 
     - do:
@@ -83,7 +83,7 @@
 ---
 "Filtered templates":
     - requires:
-        cluster_features: [ "cat_templates_v2" ]
+        cluster_features: [ "templates_v2" ]
         reason: "format changed in 7.8 to accomodate V2 index templates"
 
     - do:
@@ -125,7 +125,7 @@
 ---
 "Column headers":
     - requires:
-        cluster_features: [ "cat_templates_v2" ]
+        cluster_features: [ "templates_v2" ]
         reason: "format changed in 7.8 to accomodate V2 index templates"
 
     - do:
@@ -163,7 +163,7 @@
 ---
 "Select columns":
     - requires:
-        cluster_features: [ "cat_templates_v2" ]
+        cluster_features: [ "templates_v2" ]
         reason: "format changed in 7.8 to accomodate V2 index templates"
 
     - do:
@@ -197,7 +197,7 @@
 ---
 "Sort templates":
     - requires:
-        cluster_features: [ "cat_templates_v2" ]
+        cluster_features: [ "templates_v2" ]
         reason: "format changed in 7.8 to accomodate V2 index templates"
         test_runner_features: default_shards, no_xpack
 
@@ -250,7 +250,7 @@
 ---
 "Multiple template":
     - requires:
-        cluster_features: [ "cat_templates_v2" ]
+        cluster_features: [ "templates_v2" ]
         reason: "format changed in 7.8 to accomodate V2 index templates"
         test_runner_features: default_shards, no_xpack
 
@@ -286,7 +286,7 @@
 ---
 "Mixture of legacy and composable templates":
     - requires:
-        cluster_features: [ "cat_templates_v2" ]
+        cluster_features: [ "templates_v2" ]
         reason: "format changed in 7.8 to accomodate V2 index templates"
         test_runner_features: allowed_warnings
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml
index 1f0e2b6fd727c..8c350b50a6bf2 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yml
@@ -49,10 +49,10 @@
 
 ---
 "Cluster shard allocation explanation test with a closed index":
-  - skip:
-      version: " - 7.1.99"
+  - requires:
+      cluster_features: ["indices_replicate_closed"]
       reason: closed indices are replicated starting version 7.2.0
-      features: ["allowed_warnings"]
+      test_runner_features: ["allowed_warnings"]
 
   - do:
       indices.create:
@@ -95,8 +95,8 @@
 
 ---
 "Cluster allocation explanation response includes node's roles":
-  - skip:
-      version: " - 8.10.99"
+  - requires:
+      cluster_features: ["cluster_allocation_role"]
       reason: The roles field was introduced in 8.11.0
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
index d27abc3da7081..0308a68dae2cd 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml
@@ -1,7 +1,7 @@
 ---
 "Basic CRUD":
-  - skip:
-      version: " - 7.7.99"
+  - requires:
+      cluster_features: ["templates_v2"]
       reason: "index/component template v2 API unavailable before 7.8"
 
   - do:
@@ -48,8 +48,8 @@
 
 ---
 "Delete multiple templates":
-  - skip:
-      version: " - 7.99.99"
+  - requires:
+      cluster_features: ["cluster_templates_delete_multiple"]
       reason: "not yet backported"
 
   - do:
@@ -116,8 +116,8 @@
 
 ---
 "Add data stream lifecycle":
-  - skip:
-      version: " - 8.10.99"
+  - requires:
+      cluster_features: ["datastream_lifecycle"]
       reason: "Data stream lifecycle was available from 8.11"
 
   - do:
@@ -144,8 +144,8 @@
 
 ---
 "Get data stream lifecycle with default rollover":
-  - skip:
-      version: " - 8.10.99"
+  - requires:
+      cluster_features: ["datastream_lifecycle"]
       reason: "Data stream lifecycle was available from 8.11"
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_balance/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_balance/10_basic.yml
index a4204034bfd80..cd213ebe72a8e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_balance/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_balance/10_basic.yml
@@ -1,7 +1,7 @@
 ---
 setup:
-  - skip:
-      version: " - 8.5.99"
+  - requires:
+      cluster_features: ["cluster_desired_balance"]
       reason: "API added in in 8.6.0"
 
 ---
@@ -61,8 +61,8 @@ setup:
 ---
 "Test cluster_balance_stats":
 
-  - skip:
-      version: " - 8.6.99"
+  - requires:
+      cluster_features: ["cluster_desired_balance_stats"]
       reason: "cluster_balance_stats added in in 8.7.0"
 
   - do:
@@ -109,8 +109,8 @@ setup:
 ---
 "Test cluster_info":
 
-  - skip:
-      version: " - 8.7.99"
+  - requires:
+      cluster_features: ["cluster_info"]
       reason: "cluster_info added in in 8.8.0"
 
   - do:
@@ -121,8 +121,8 @@ setup:
 ---
 "Test cluster_balance_stats contains node ID and roles":
 
-  - skip:
-      version: " - 8.7.99"
+  - requires:
+      cluster_features: ["cluster_desired_balance_extended"]
       reason: "node_id and roles added in in 8.8.0"
 
   - do:
@@ -140,8 +140,8 @@ setup:
 ---
 "Test tier_preference":
 
-  - skip:
-      version: " - 8.7.99"
+  - requires:
+      cluster_features: ["cluster_desired_balance_extended"]
       reason: "tier_preference added in in 8.8.0"
 
   - do:
@@ -165,8 +165,8 @@ setup:
 ---
 "Test computed_shard_movements":
 
-  - skip:
-      version: " - 8.7.99"
+  - requires:
+      cluster_features: ["cluster_desired_balance_extended"]
       reason: "computed_shard_movements added in in 8.8.0"
 
   - do:
@@ -177,8 +177,8 @@ setup:
 ---
 "Test reset desired balance":
 
-  - skip:
-      version: " - 8.7.99"
+  - requires:
+      cluster_features: ["cluster_desired_balance_extended"]
       reason: "reset API added in in 8.8.0"
 
   - do:
@@ -187,8 +187,8 @@ setup:
 ---
 "Test undesired_shard_allocation_count":
 
-  - skip:
-      version: " - 8.11.99"
+  - requires:
+      cluster_features: ["cluster_desired_balance_stats_undesired_count"]
       reason: "undesired_shard_allocation_count added in in 8.12.0"
 
   - do:
@@ -225,8 +225,8 @@ setup:
 ---
 "Test unassigned_shards, total_allocations, undesired_allocations and undesired_allocations_fraction":
 
-  - skip:
-      version: " - 8.11.99"
+  - requires:
+      cluster_features: [ "cluster_desired_balance_stats_undesired_count" ]
       reason: "undesired_shard_allocation_count added in in 8.12.0"
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/20_dry_run.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/20_dry_run.yml
index 3723c5b2165ca..56a693febec82 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/20_dry_run.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/20_dry_run.yml
@@ -1,7 +1,7 @@
 ---
 setup:
-  - skip:
-      version: " - 8.3.99"
+  - requires:
+      cluster_features: ["cluster_desired_nodes_dry_run"]
       reason: "Support for the dry run option was added in in 8.4.0"
 ---
 teardown:
@@ -12,7 +12,7 @@ teardown:
 "Test dry run doesn't update empty desired nodes":
   - skip:
       version: " - 8.12.99"
-      reason: "version_node removed from version 8.13 onwards"
+      reason: "version_node removed from version 8.13 onwards, require the new api"
   - do:
       cluster.state: {}
 
@@ -42,7 +42,7 @@ teardown:
 "Test dry run doesn't update existing desired nodes":
   - skip:
       version: " - 8.12.99"
-      reason: "version_node removed from version 8.13 onwards"
+      reason: "version_node removed from version 8.13 onwards, require the new api"
   - do:
       cluster.state: {}
 
@@ -94,7 +94,7 @@ teardown:
 ---
 "Test validation works for dry run updates":
   - skip:
-      version: "8.9.99 - "
+      cluster_features: ["cluster_desired_nodes_no_settings_validation"]
       reason: "We started skipping setting validations in 8.10"
   - do:
       cluster.state: { }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml
index a051b3626b217..586bd3b7cfb6b 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml
@@ -132,9 +132,10 @@
 ---
 "cluster health with closed index (pre 7.2.0)":
   - skip:
-      version: "7.2.0 - "
+      cluster_features: ["indices_replicate_closed"]
       reason:  "closed indices are replicated starting version 7.2.0"
-      features: ["allowed_warnings"]
+  - requires:
+      test_runner_features: ["allowed_warnings"]
 
   - do:
       indices.create:
@@ -208,10 +209,10 @@
 
 ---
 "cluster health with closed index":
-  - skip:
-      version: " - 7.1.99"
+  - requires:
+      cluster_features: ["indices_replicate_closed"]
       reason:  "closed indices are replicated starting version 7.2.0"
-      features: ["allowed_warnings", "default_shards"]
+      test_runner_features: ["allowed_warnings", "default_shards"]
 
   - do:
       indices.create:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/30_indices_options.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/30_indices_options.yml
index 8144a2c1a1dbf..5caa0ebad30b2 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/30_indices_options.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/30_indices_options.yml
@@ -34,8 +34,8 @@ setup:
 
 ---
 "cluster health with expand_wildcards":
-  - skip:
-      version: " - 7.1.99"
+  - requires:
+      cluster_features: ["cluster_health_indices_options"]
       reason:  "indices options has been introduced in cluster health request starting version 7.2.0"
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/10_info_all.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/10_info_all.yml
index 4e88c14a2b1dc..023e35cd4bee1 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/10_info_all.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/10_info_all.yml
@@ -1,7 +1,7 @@
 ---
 setup:
-  - skip:
-      version: " - 8.8.99"
+  - requires:
+      cluster_features: ["cluster_info_extended"]
       reason: "/_info/_all only available from v8.9"
 
 ---
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/20_info_http.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/20_info_http.yml
index f238e5116e146..ff7d982b14fee 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/20_info_http.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/20_info_http.yml
@@ -1,7 +1,7 @@
 ---
 "Cluster HTTP Info":
-  - skip:
-      version: " - 8.8.99"
+  - requires:
+      cluster_features: ["cluster_info_extended"]
       reason: "/_info/http only available from v8.9"
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/30_info_thread_pool.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/30_info_thread_pool.yml
index 35a538ac3a44a..261f1d8ea56cb 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/30_info_thread_pool.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/30_info_thread_pool.yml
@@ -1,7 +1,7 @@
 ---
 "Cluster HTTP Info":
-  - skip:
-      version: " - 8.8.99"
+  - requires:
+      cluster_features: ["cluster_info_extended"]
       reason: "/_info/thread_pool only available from v8.9"
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/40_info_script.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/40_info_script.yml
index 8c4b3e2832daf..bb7597537014e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/40_info_script.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.info/40_info_script.yml
@@ -1,7 +1,7 @@
 ---
 "Cluster HTTP Info":
-  - skip:
-      version: " - 8.8.99"
+  - requires:
+      cluster_features: ["cluster_info_extended"]
       reason: "/_info/script only available from v8.9"
 
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.prevalidate_node_removal/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.prevalidate_node_removal/10_basic.yml
index 740836efcdc46..fda715e416ac2 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.prevalidate_node_removal/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.prevalidate_node_removal/10_basic.yml
@@ -1,8 +1,8 @@
 ---
 "Prevalidation basic test":
-  - skip:
-      features: contains
-      version: "- 8.6.99"
+  - requires:
+      test_runner_features: ["contains"]
+      cluster_features: ["cluster_prevalidate_node_removal_reason"]
       reason: "The reason field was introduced in 8.7.0"
 
   # Fetch a node ID and stash it in node_id
@@ -19,8 +19,8 @@
   - contains: {nodes: {id: "$node_id", result: {is_safe: true, reason: no_problems, message: ""}}}
 ---
 "Prevalidation with no node specified":
-  - skip:
-      version: "- 8.5.99"
+  - requires:
+      cluster_features: ["cluster_prevalidate_node_removal"]
       reason: "API added in 8.6.0"
   - do:
       catch: bad_request
@@ -31,8 +31,8 @@
   - match: { error.reason: "Validation Failed: 1: request must contain one of the parameters 'names', 'ids', or 'external_ids';" }
 ---
 "Prevalidation with more than one query parameter":
-  - skip:
-      version: "- 8.5.99"
+  - requires:
+      cluster_features: ["cluster_prevalidate_node_removal"]
       reason: "API added in 8.6.0"
   - do:
       catch: bad_request
@@ -45,8 +45,8 @@
   - match: { error.reason: "Validation Failed: 1: request must contain only one of the parameters 'names', 'ids', or 'external_ids';" }
 ---
 "Prevalidation with non-existing node":
-  - skip:
-      version: "- 8.5.99"
+  - requires:
+      cluster_features: ["cluster_prevalidate_node_removal"]
       reason: "API added in 8.6.0"
   - do:
       catch: missing
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml
index a46d9a67a863d..2c48282332909 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml
@@ -75,8 +75,8 @@
 ---
 "get cluster stats returns packaging types":
 
-  - skip:
-      version: " - 7.1.99"
+  - requires:
+      cluster_features: ["cluster_stats_packaging_types"]
       reason:  "packaging types are added for v7.2.0"
 
   - do:
@@ -85,8 +85,8 @@
   - is_true: nodes.packaging_types
 ---
 "get cluster stats without runtime fields":
-  - skip:
-      version: " - 7.12.99"
+  - requires:
+      cluster_features: ["cluster_stats_runtime_fields"]
       reason:  "cluster stats includes runtime fields from 7.13 on"
   - do:
       indices.create:
@@ -98,8 +98,8 @@
 
 ---
 "Usage stats with script-less runtime fields":
-  - skip:
-      version: " - 7.12.99"
+  - requires:
+      cluster_features: ["cluster_stats_runtime_fields"]
       reason:  "cluster stats includes runtime fields from 7.13 on"
   - do:
       indices.create:
@@ -168,8 +168,8 @@
 
 ---
 "mappings sizes reported in get cluster stats":
-  - skip:
-      version: " - 8.3.99"
+  - requires:
+      cluster_features: ["cluster_stats_mapping_sizes"]
       reason:  "mapping sizes reported from 8.4 onwards"
   - do:
       indices.create:
@@ -188,8 +188,8 @@
 
 ---
 "snapshot stats reported in get cluster stats":
-  - skip:
-      version: " - 8.7.99"
+  - requires:
+      cluster_features: ["cluster_stats_snapshots"]
       reason:  "snapshot stats reported from 8.8 onwards"
 
   - do:
@@ -231,8 +231,8 @@
 
 ---
 "Dense vector stats":
-  - skip:
-      version: " - 8.9.99"
+  - requires:
+      cluster_features: ["cluster_stats_dense_vectors"]
       reason:  "dense vector stats added in 8.10"
   - do:
       indices.create:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/20_indexing_pressure.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/20_indexing_pressure.yml
index 9178fab25e230..648964d9e721f 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/20_indexing_pressure.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/20_indexing_pressure.yml
@@ -1,7 +1,7 @@
 ---
 "Indexing pressure cluster stats":
-  - skip:
-      version: " - 8.0.99"
+  - requires:
+      cluster_features: ["cluster_stats_indexing_pressure"]
       reason: "indexing_pressure in cluster was added in 8.1"
 
   - do:
diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/YamlTestLegacyFeatures.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/YamlTestLegacyFeatures.java
index 326afdaa7ae1a..2ce35888c3f14 100644
--- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/YamlTestLegacyFeatures.java
+++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/YamlTestLegacyFeatures.java
@@ -19,7 +19,6 @@
  * to support BwC. Rather than leaving them in the main src we group them here, so it's clear they are not used in production code anymore.
  */
 public class YamlTestLegacyFeatures implements FeatureSpecification {
-
     private static final NodeFeature BULK_AUTO_ID = new NodeFeature("bulk_auto_id");
     private static final NodeFeature BULK_REQUIRE_ALIAS = new NodeFeature("bulk_require_alias");
     private static final NodeFeature BULK_DYNAMIC_TEMPLATE_OP_TYPE = new NodeFeature("bulk_dynamic_template_op_type");
@@ -36,7 +35,6 @@ public class YamlTestLegacyFeatures implements FeatureSpecification {
     private static final NodeFeature CAT_ALLOCATION_NODE_ROLE = new NodeFeature("cat_allocation_node_role");
 
     private static final NodeFeature CAT_INDICES_DATASET_SIZE = new NodeFeature("cat_indices_dataset_size");
-    private static final NodeFeature CAT_INDICES_REPLICATE_CLOSED = new NodeFeature("cat_indices_replicate_closed");
     private static final NodeFeature CAT_INDICES_VALIDATE_HEALTH_PARAM = new NodeFeature("cat_indices_validate_health_param");
 
     private static final NodeFeature CAT_PLUGINS_NEW_FORMAT = new NodeFeature("cat_plugins_new_format");
@@ -48,9 +46,47 @@ public class YamlTestLegacyFeatures implements FeatureSpecification {
 
     private static final NodeFeature CAT_TASKS_X_OPAQUE_ID = new NodeFeature("cat_tasks_x_opaque_id");
 
-    private static final NodeFeature CAT_TEMPLATES_V2 = new NodeFeature("cat_templates_v2");
     private static final NodeFeature CAT_TEMPLATE_NAME_VALIDATION = new NodeFeature("cat_template_name_validation");
 
+    private static final NodeFeature CLUSTER_TEMPLATES_DELETE_MULTIPLE = new NodeFeature("cluster_templates_delete_multiple");
+
+    private static final NodeFeature CLUSTER_ALLOCATION_ROLE = new NodeFeature("cluster_allocation_role");
+
+    private static final NodeFeature CLUSTER_DESIRED_BALANCE = new NodeFeature("cluster_desired_balance");
+    private static final NodeFeature CLUSTER_DESIRED_BALANCE_STATS = new NodeFeature("cluster_desired_balance_stats");
+    private static final NodeFeature CLUSTER_DESIRED_BALANCE_EXTENDED = new NodeFeature("cluster_desired_balance_extended");
+    private static final NodeFeature CLUSTER_DESIRED_BALANCE_STATS_UNDESIRED_COUNT = new NodeFeature(
+        "cluster_desired_balance_stats_undesired_count"
+    );
+
+    private static final NodeFeature CLUSTER_DESIRED_NODES_OLD = new NodeFeature("cluster_desired_nodes_old");
+    private static final NodeFeature CLUSTER_DESIRED_NODES_DRY_RUN = new NodeFeature("cluster_desired_nodes_dry_run");
+    private static final NodeFeature CLUSTER_DESIRED_NODES_NO_SETTINGS_VALIDATION = new NodeFeature(
+        "cluster_desired_nodes_no_settings_validation"
+    );
+    private static final NodeFeature CLUSTER_DESIRED_NODES = new NodeFeature("cluster_desired_nodes");
+
+    private static final NodeFeature CLUSTER_HEALTH_INDICES_OPTIONS = new NodeFeature("cluster_health_indices_options");
+
+    private static final NodeFeature CLUSTER_INFO = new NodeFeature("cluster_info");
+    private static final NodeFeature CLUSTER_INFO_EXTENDED = new NodeFeature("cluster_info_extended");
+
+    private static final NodeFeature CLUSTER_PREVALIDATE_NODE_REMOVAL = new NodeFeature("cluster_prevalidate_node_removal");
+    private static final NodeFeature CLUSTER_PREVALIDATE_NODE_REMOVAL_REASON = new NodeFeature("cluster_prevalidate_node_removal_reason");
+
+    private static final NodeFeature CLUSTER_STATS_PACKAGING_TYPES = new NodeFeature("cluster_stats_packaging_types");
+    private static final NodeFeature CLUSTER_STATS_RUNTIME_FIELDS = new NodeFeature("cluster_stats_runtime_fields");
+    private static final NodeFeature CLUSTER_STATS_INDEXING_PRESSURE = new NodeFeature("cluster_stats_indexing_pressure");
+    private static final NodeFeature CLUSTER_STATS_MAPPING_SIZES = new NodeFeature("cluster_stats_mapping_sizes");
+    private static final NodeFeature CLUSTER_STATS_SNAPSHOTS = new NodeFeature("cluster_stats_snapshots");
+    private static final NodeFeature CLUSTER_STATS_DENSE_VECTORS = new NodeFeature("cluster_stats_dense_vectors");
+
+    private static final NodeFeature DATASTREAM_LIFECYCLE = new NodeFeature("datastream_lifecycle");
+
+    private static final NodeFeature TEMPLATES_V2 = new NodeFeature("templates_v2");
+
+    private static final NodeFeature INDICES_REPLICATE_CLOSED = new NodeFeature("indices_replicate_closed");
+
     @Override
     public Map<NodeFeature, Version> getHistoricalFeatures() {
         return Map.ofEntries(
@@ -66,7 +102,6 @@ public Map<NodeFeature, Version> getHistoricalFeatures() {
 
             Map.entry(CAT_ALLOCATION_NODE_ROLE, Version.V_8_10_0),
 
-            Map.entry(CAT_INDICES_REPLICATE_CLOSED, Version.V_7_2_0),
             Map.entry(CAT_INDICES_VALIDATE_HEALTH_PARAM, Version.V_7_8_0),
             Map.entry(CAT_INDICES_DATASET_SIZE, Version.V_8_11_0),
 
@@ -79,8 +114,40 @@ public Map<NodeFeature, Version> getHistoricalFeatures() {
 
             Map.entry(CAT_TASKS_X_OPAQUE_ID, Version.V_7_10_0),
 
-            Map.entry(CAT_TEMPLATES_V2, Version.V_7_8_0),
-            Map.entry(CAT_TEMPLATE_NAME_VALIDATION, Version.V_7_16_0)
+            Map.entry(CAT_TEMPLATE_NAME_VALIDATION, Version.V_7_16_0),
+
+            Map.entry(CLUSTER_TEMPLATES_DELETE_MULTIPLE, Version.V_8_0_0),
+            Map.entry(CLUSTER_ALLOCATION_ROLE, Version.V_8_11_0),
+
+            Map.entry(CLUSTER_DESIRED_BALANCE, Version.V_8_6_0),
+            Map.entry(CLUSTER_DESIRED_BALANCE_STATS, Version.V_8_7_0),
+            Map.entry(CLUSTER_DESIRED_BALANCE_EXTENDED, Version.V_8_8_0),
+            Map.entry(CLUSTER_DESIRED_BALANCE_STATS_UNDESIRED_COUNT, Version.V_8_12_0),
+
+            Map.entry(CLUSTER_DESIRED_NODES_OLD, Version.V_8_3_0),
+            Map.entry(CLUSTER_DESIRED_NODES_DRY_RUN, Version.V_8_4_0),
+            Map.entry(CLUSTER_DESIRED_NODES_NO_SETTINGS_VALIDATION, Version.V_8_10_0),
+
+            Map.entry(CLUSTER_HEALTH_INDICES_OPTIONS, Version.V_7_2_0),
+
+            Map.entry(CLUSTER_INFO, Version.V_8_8_0),
+            Map.entry(CLUSTER_INFO_EXTENDED, Version.V_8_9_0),
+
+            Map.entry(CLUSTER_PREVALIDATE_NODE_REMOVAL, Version.V_8_6_0),
+            Map.entry(CLUSTER_PREVALIDATE_NODE_REMOVAL_REASON, Version.V_8_7_0),
+
+            Map.entry(CLUSTER_STATS_PACKAGING_TYPES, Version.V_7_2_0),
+            Map.entry(CLUSTER_STATS_RUNTIME_FIELDS, Version.V_7_13_0),
+            Map.entry(CLUSTER_STATS_INDEXING_PRESSURE, Version.V_8_1_0),
+            Map.entry(CLUSTER_STATS_MAPPING_SIZES, Version.V_8_4_0),
+            Map.entry(CLUSTER_STATS_SNAPSHOTS, Version.V_8_8_0),
+            Map.entry(CLUSTER_STATS_DENSE_VECTORS, Version.V_8_10_0),
+
+            Map.entry(DATASTREAM_LIFECYCLE, Version.V_8_11_0),
+
+            Map.entry(INDICES_REPLICATE_CLOSED, Version.V_7_2_0),
+
+            Map.entry(TEMPLATES_V2, Version.V_7_8_0)
         );
     }
 }

From 1fe4946f81fcc08c23ceccaf94bfacd36846b17e Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 22 Mar 2024 08:18:48 +0000
Subject: [PATCH 06/31] Fix testScheduledFixedDelayRejection (#106630)

Not really necessary to allow the scheduled task to race against the
blocks, and this race is a source of test flakiness. Fixed by imposing
the blocks first.

Closes #106618
---
 .../org/elasticsearch/threadpool/ThreadPoolTests.java  | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
index 8a61054f5fa93..66d3dd7a829eb 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
@@ -465,7 +465,6 @@ public void testScheduledOneShotForceExecution() {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106618")
     public void testScheduledFixedDelayRejection() {
         final var name = "fixed-bounded";
         final var threadPool = new TestThreadPool(
@@ -476,17 +475,14 @@ public void testScheduledFixedDelayRejection() {
         final var future = new PlainActionFuture<Void>();
         final var latch = new CountDownLatch(1);
         try {
+            blockExecution(threadPool.executor(name), latch);
             threadPool.scheduleWithFixedDelay(
-                ActionRunnable.wrap(future, ignored -> Thread.yield()),
+                ActionRunnable.wrap(future, ignored -> fail("should not execute")),
                 TimeValue.timeValueMillis(between(1, 100)),
                 threadPool.executor(name)
             );
 
-            while (future.isDone() == false) {
-                // might not block all threads the first time round if the scheduled runnable is running, so must keep trying
-                blockExecution(threadPool.executor(name), latch);
-            }
-            expectThrows(EsRejectedExecutionException.class, () -> FutureUtils.get(future));
+            expectThrows(EsRejectedExecutionException.class, () -> FutureUtils.get(future, 10, TimeUnit.SECONDS));
         } finally {
             latch.countDown();
             assertTrue(terminate(threadPool));

From cc8fb4dba9536c33abf6a4f396657ba97e52518c Mon Sep 17 00:00:00 2001
From: Alexander Spies <alexander.spies@elastic.co>
Date: Fri, 22 Mar 2024 09:42:29 +0100
Subject: [PATCH 07/31] ESQL: Re-enable logical dependency check (#105860)

* Fix Enrich refs locally in the dependency check
* Re-enable logical dependency check + test for it
* Return server error on failed dependency check
---
 docs/changelog/105860.yaml                             |  5 +++++
 .../xpack/esql/optimizer/LogicalVerifier.java          |  9 ++++++---
 .../xpack/esql/optimizer/OptimizerRules.java           | 10 ++++++++++
 .../esql/optimizer/LogicalPlanOptimizerTests.java      |  3 +--
 4 files changed, 22 insertions(+), 5 deletions(-)
 create mode 100644 docs/changelog/105860.yaml

diff --git a/docs/changelog/105860.yaml b/docs/changelog/105860.yaml
new file mode 100644
index 0000000000000..71f3544a02a1f
--- /dev/null
+++ b/docs/changelog/105860.yaml
@@ -0,0 +1,5 @@
+pr: 105860
+summary: "ESQL: Re-enable logical dependency check"
+area: ES|QL
+type: bug
+issues: []
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java
index bf569ee587dbc..6b62029bd8f45 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java
@@ -22,11 +22,10 @@ private LogicalVerifier() {}
     /** Verifies the optimized logical plan. */
     public Failures verify(LogicalPlan plan) {
         Failures failures = new Failures();
+        Failures dependencyFailures = new Failures();
 
         plan.forEachUp(p -> {
-            // dependency check
-            // FIXME: re-enable
-            // DEPENDENCY_CHECK.checkPlan(p, failures);
+            DEPENDENCY_CHECK.checkPlan(p, dependencyFailures);
 
             if (failures.hasFailures() == false) {
                 p.forEachExpression(ex -> {
@@ -37,6 +36,10 @@ public Failures verify(LogicalPlan plan) {
             }
         });
 
+        if (dependencyFailures.hasFailures()) {
+            throw new IllegalStateException(dependencyFailures.toString());
+        }
+
         return failures;
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
index 645924907b6f5..e375f11ab3ae7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
@@ -79,6 +79,16 @@ protected AttributeSet generates(P p) {
     }
 
     static class LogicalPlanDependencyCheck extends DependencyConsistency<LogicalPlan> {
+        @Override
+        protected AttributeSet references(LogicalPlan plan) {
+            if (plan instanceof Enrich enrich) {
+                // The enrichFields are NamedExpressions, so we compute their references as well when just calling enrich.references().
+                // But they are not actually referring to attributes from the input plan - only the match field does.
+                return enrich.matchField().references();
+            }
+            return super.references(plan);
+        }
+
         @Override
         protected AttributeSet generates(LogicalPlan logicalPlan) {
             // source-like operators
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
index c6747c9d65d24..952fbc6f265e4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
@@ -3299,7 +3299,6 @@ public void testEmptyMappingIndex() {
         assertThat(Expressions.names(local.output()), contains(NO_FIELDS.get(0).name(), "x", "language_code", "language_name"));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105436")
     public void testPlanSanityCheck() throws Exception {
         var plan = optimizedPlan("""
             from test
@@ -3325,7 +3324,7 @@ public void testPlanSanityCheck() throws Exception {
             )
         );
 
-        VerificationException e = expectThrows(VerificationException.class, () -> logicalOptimizer.optimize(invalidPlan));
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> logicalOptimizer.optimize(invalidPlan));
         assertThat(e.getMessage(), containsString("Plan [OrderBy[[Order[salary"));
         assertThat(e.getMessage(), containsString(" optimized incorrectly due to missing references [salary"));
     }

From b5923eb8dc42d11eb2d44ea8802008066ebc7344 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 22 Mar 2024 08:56:15 +0000
Subject: [PATCH 08/31] AwaitsFix for #106650

---
 .../test/ml/search_knn_query_vector_builder.yml             | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/search_knn_query_vector_builder.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/search_knn_query_vector_builder.yml
index 97e5146e9af86..50f687f704994 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/search_knn_query_vector_builder.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/search_knn_query_vector_builder.yml
@@ -106,8 +106,10 @@ setup:
 ---
 "Test vector search with query_vector_builder":
   - skip:
-      version: " - 8.13.99"
-      reason: "introduced after 8.13"
+      version: all
+      reason: AwaitsFix https://github.com/elastic/elasticsearch/issues/106650
+#      version: " - 8.13.99"
+#      reason: "introduced after 8.13"
   - do:
       search:
         index: index-with-embedded-text

From 23278a52cf60e9d155b4423c8859379c7130d8bc Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 22 Mar 2024 09:50:18 +0000
Subject: [PATCH 09/31] Expand conceptual docs around `ActionListener`
 (#105405)

Rephrases the docs about `ActionListener` in terms of continuations and
control flow, rather than as injecting logic into lower layers.
---
 docs/internal/DistributedArchitectureGuide.md | 72 ++++++++++++++++---
 1 file changed, 61 insertions(+), 11 deletions(-)

diff --git a/docs/internal/DistributedArchitectureGuide.md b/docs/internal/DistributedArchitectureGuide.md
index ea5942ff71cc8..a89956721a481 100644
--- a/docs/internal/DistributedArchitectureGuide.md
+++ b/docs/internal/DistributedArchitectureGuide.md
@@ -10,20 +10,70 @@
 
 ### ActionListener
 
-`ActionListener`s are a means off injecting logic into lower layers of the code. They encapsulate a block of code that takes a response
-value -- the `onResponse()` method --, and then that block of code (the `ActionListener`) is passed into a function that will eventually
-execute the code (call `onResponse()`) when a response value is available. `ActionListener`s are used to pass code down to act on a result,
-rather than lower layers returning a result back up to be acted upon by the caller. One of three things can happen to a listener: it can be
-executed in the same thread — e.g. `ActionListener.run()` --; it can be passed off to another thread to be executed; or it can be added to
-a list someplace, to eventually be executed by some service. `ActionListener`s also define `onFailure()` logic, in case an error is
-encountered before a result can be formed.
+Callbacks are used extensively throughout Elasticsearch because they enable us to write asynchronous and nonblocking code, i.e. code which
+doesn't necessarily compute a result straight away but also doesn't block the calling thread waiting for the result to become available.
+They support several useful control flows:
+
+- They can be completed immediately on the calling thread.
+- They can be completed concurrently on a different thread.
+- They can be stored in a data structure and completed later on when the system reaches a particular state.
+- Most commonly, they can be passed on to other methods that themselves require a callback.
+- They can be wrapped in another callback which modifies the behaviour of the original callback, perhaps adding some extra code to run
+  before or after completion, before passing them on.
+
+`ActionListener` is a general-purpose callback interface that is used extensively across the Elasticsearch codebase. `ActionListener` is
+used pretty much everywhere that needs to perform some asynchronous and nonblocking computation. The uniformity makes it easier to compose
+parts of the system together without needing to build adapters to convert back and forth between different kinds of callback. It also makes
+it easier to develop the skills needed to read and understand all the asynchronous code, although this definitely takes practice and is
+certainly not easy in an absolute sense. Finally, it has allowed us to build a rich library for working with `ActionListener` instances
+themselves, creating new instances out of existing ones and completing them in interesting ways. See for instance:
+
+- all the static methods on [ActionListener](https://github.com/elastic/elasticsearch/blob/v8.12.2/server/src/main/java/org/elasticsearch/action/ActionListener.java) itself
+- [`ThreadedActionListener`](https://github.com/elastic/elasticsearch/blob/v8.12.2/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java) for forking work elsewhere
+- [`RefCountingListener`](https://github.com/elastic/elasticsearch/blob/v8.12.2/server/src/main/java/org/elasticsearch/action/support/RefCountingListener.java) for running work in parallel
+- [`SubscribableListener`](https://github.com/elastic/elasticsearch/blob/v8.12.2/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java) for constructing flexible workflows
+
+Callback-based asynchronous code can easily call regular synchronous code, but synchronous code cannot run callback-based asynchronous code
+without blocking the calling thread until the callback is called back. This blocking is at best undesirable (threads are too expensive to
+waste with unnecessary blocking) and at worst outright broken (the blocking can lead to deadlock). Unfortunately this means that most of our
+code ends up having to be written with callbacks, simply because it's ultimately calling into some other code that takes a callback. The
+entry points for all Elasticsearch APIs are callback-based (e.g. REST APIs all start at
+[`org.elasticsearch.rest.BaseRestHandler#prepareRequest`](https://github.com/elastic/elasticsearch/blob/v8.12.2/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java#L158-L171),
+and transport APIs all start at
+[`org.elasticsearch.action.support.TransportAction#doExecute`](https://github.com/elastic/elasticsearch/blob/v8.12.2/server/src/main/java/org/elasticsearch/action/support/TransportAction.java#L65))
+and the whole system fundamentally works in terms of an event loop (a `io.netty.channel.EventLoop`) which processes network events via
+callbacks.
+
+`ActionListener` is not an _ad-hoc_ invention. Formally speaking, it is our implementation of the general concept of a continuation in the
+sense of [_continuation-passing style_](https://en.wikipedia.org/wiki/Continuation-passing_style) (CPS): an extra argument to a function
+which defines how to continue the computation when the result is available. This is in contrast to _direct style_ which is the more usual
+style of calling methods that return values directly back to the caller so they can continue executing as normal. There's essentially two
+ways that computation can continue in Java (it can return a value or it can throw an exception) which is why `ActionListener` has both an
+`onResponse()` and an `onFailure()` method.
+
+CPS is strictly more expressive than direct style: direct code can be mechanically translated into continuation-passing style, but CPS also
+enables all sorts of other useful control structures such as forking work onto separate threads, possibly to be executed in parallel,
+perhaps even across multiple nodes, or possibly collecting a list of continuations all waiting for the same condition to be satisfied before
+proceeding (e.g.
+[`SubscribableListener`](https://github.com/elastic/elasticsearch/blob/v8.12.2/server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java)
+amongst many others). Some languages have first-class support for continuations (e.g. the `async` and `await` primitives in C#) allowing the
+programmer to write code in direct style away from those exotic control structures, but Java does not. That's why we have to manipulate all
+the callbacks ourselves.
+
+Strictly speaking, CPS requires that a computation _only_ continues by calling the continuation. In Elasticsearch, this means that
+asynchronous methods must have `void` return type and may not throw any exceptions. This is mostly the case in our code as written today,
+and is a good guiding principle, but we don't enforce void exceptionless methods and there are some deviations from this rule. In
+particular, it's not uncommon to permit some methods to throw an exception, using things like
+[`ActionListener#run`](https://github.com/elastic/elasticsearch/blob/v8.12.2/server/src/main/java/org/elasticsearch/action/ActionListener.java#L381-L390)
+(or an equivalent `try ... catch ...` block) further up the stack to handle it. Some methods also take (and may complete) an
+`ActionListener` parameter, but still return a value separately for other local synchronous work.
 
 This pattern is often used in the transport action layer with the use of the
-[ChannelActionListener]([url](https://github.com/elastic/elasticsearch/blob/8.12/server/src/main/java/org/elasticsearch/action/support/ChannelActionListener.java))
+[ChannelActionListener](https://github.com/elastic/elasticsearch/blob/v8.12.2/server/src/main/java/org/elasticsearch/action/support/ChannelActionListener.java)
 class, which wraps a `TransportChannel` produced by the transport layer. `TransportChannel` implementations can hold a reference to a Netty
-channel with which to pass the response back to the network caller. Netty has a many-to-one association of network callers to channels, so
-a call taking a long time generally won't hog resources: it's cheap. A transport action can take hours to respond and that's alright,
-barring caller timeouts.
+channel with which to pass the response back to the network caller. Netty has a many-to-one association of network callers to channels, so a
+call taking a long time generally won't hog resources: it's cheap. A transport action can take hours to respond and that's alright, barring
+caller timeouts.
 
 (TODO: add useful starter references and explanations for a range of Listener classes. Reference the Netty section.)
 

From 4647691809b56892a8ab6b6e93e6a5dcfae09a3e Mon Sep 17 00:00:00 2001
From: Ievgen Degtiarenko <ievgen.degtiarenko@elastic.co>
Date: Fri, 22 Mar 2024 11:00:39 +0100
Subject: [PATCH 10/31] Clarify the difference between ImmutableEntry and
 Map.entry (#106588)

---
 server/src/main/java/org/elasticsearch/common/util/Maps.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/util/Maps.java b/server/src/main/java/org/elasticsearch/common/util/Maps.java
index 1b46e71dadd12..fc911793711b7 100644
--- a/server/src/main/java/org/elasticsearch/common/util/Maps.java
+++ b/server/src/main/java/org/elasticsearch/common/util/Maps.java
@@ -331,8 +331,7 @@ public static <K, V1, V2> Map<K, V2> transformValues(Map<K, V1> source, Function
 
     /**
      * An immutable implementation of {@link Map.Entry}.
-     * @param key key
-     * @param value value
+     * Unlike {@code Map.entry(...)} this implementation permits null key and value.
      */
     public record ImmutableEntry<KType, VType>(KType key, VType value) implements Map.Entry<KType, VType> {
 

From 69ecdc643b8d7b7c9482a8feeee6380cae7abb8d Mon Sep 17 00:00:00 2001
From: Nikolaj Volgushev <n1v0lg@users.noreply.github.com>
Date: Fri, 22 Mar 2024 11:26:45 +0100
Subject: [PATCH 11/31] Expose workflow restriction parsing (#106648)

Need this one for custom role parsing upstream.
---
 .../elasticsearch/xpack/core/security/authz/RoleDescriptor.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
index e9aa982a05d8b..ecbd12a7f4643 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
@@ -1623,7 +1623,7 @@ public String toString() {
             return sb.toString();
         }
 
-        static Restriction parse(String roleName, XContentParser parser) throws IOException {
+        public static Restriction parse(String roleName, XContentParser parser) throws IOException {
             // advance to the START_OBJECT token if needed
             XContentParser.Token token = parser.currentToken() == null ? parser.nextToken() : parser.currentToken();
             if (token != XContentParser.Token.START_OBJECT) {

From 14ca58c9260e3d9bca86d561d254e84ed9266ed7 Mon Sep 17 00:00:00 2001
From: Dmitry Cherniachenko <2sabio@gmail.com>
Date: Fri, 22 Mar 2024 11:27:45 +0100
Subject: [PATCH 12/31] `Sets` utility class code cleanup (#105350)

* `Sets` utility class code cleanup

- newHashSetWithExpectedSize() returns HashSet, same as newHashSet()
- haveEmptyIntersection() iterates a smaller set, same as intersection()
- sortedDifference() accepts sets with Comparable values
- replace Set.copyOf() in addToCopy() with Set.of() to avoid 1 extra copy of values
---
 .../elasticsearch/common/util/set/Sets.java   | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/util/set/Sets.java b/server/src/main/java/org/elasticsearch/common/util/set/Sets.java
index adfa5023f2b35..75e5717d41b9f 100644
--- a/server/src/main/java/org/elasticsearch/common/util/set/Sets.java
+++ b/server/src/main/java/org/elasticsearch/common/util/set/Sets.java
@@ -40,7 +40,7 @@ public static <T> HashSet<T> newHashSet(T... elements) {
         return new HashSet<>(Arrays.asList(elements));
     }
 
-    public static <E> Set<E> newHashSetWithExpectedSize(int expectedSize) {
+    public static <E> HashSet<E> newHashSetWithExpectedSize(int expectedSize) {
         return new HashSet<>(capacity(expectedSize));
     }
 
@@ -53,7 +53,17 @@ static int capacity(int expectedSize) {
         return expectedSize < 2 ? expectedSize + 1 : (int) (expectedSize / 0.75 + 1.0);
     }
 
-    public static <T> boolean haveEmptyIntersection(Set<T> left, Set<T> right) {
+    public static <T> boolean haveEmptyIntersection(Set<T> set1, Set<T> set2) {
+        final Set<T> left;
+        final Set<T> right;
+        if (set1.size() < set2.size()) {
+            left = set1;
+            right = set2;
+        } else {
+            left = set2;
+            right = set1;
+        }
+
         for (T t : left) {
             if (right.contains(t)) {
                 return false;
@@ -95,7 +105,7 @@ public static <T> Set<T> difference(Set<T> left, Set<T> right) {
      * @param <T>   the type of the elements of the sets
      * @return the sorted relative complement of the left set with respect to the right set
      */
-    public static <T> SortedSet<T> sortedDifference(final Set<T> left, final Set<T> right) {
+    public static <T extends Comparable<T>> SortedSet<T> sortedDifference(final Set<T> left, final Set<T> right) {
         final SortedSet<T> set = new TreeSet<>();
         for (T k : left) {
             if (right.contains(k) == false) {
@@ -165,11 +175,12 @@ public static <T> Set<T> intersection(Set<T> set1, Set<T> set2) {
      *
      * @param set      set to copy
      * @param elements elements to add
+     * @return the unmodifiable copy of the input set with the extra elements added
      */
     @SuppressWarnings("unchecked")
     public static <E> Set<E> addToCopy(Set<E> set, E... elements) {
         final var res = new HashSet<>(set);
         Collections.addAll(res, elements);
-        return Set.copyOf(res);
+        return (Set<E>) Set.of(res.toArray());
     }
 }

From d8fc8779c469e688336a23bd34f91ce8af9f417c Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 22 Mar 2024 12:06:27 +0100
Subject: [PATCH 13/31] Fix
 S3RepositoryThirdPartyTests.testReadFromPositionLargerThanBlobLength
 (#106466)

The test should use a random operation purpose that is not "Indices", otherwise S3RetryingInputStream retries up to Integer.MAX_VALUE times which causes the test suite to timeout.

Also fixes the progress in the retries log messages.

Closes #106457
---
 .../s3/S3RepositoryThirdPartyTests.java       | 28 ++++++++++++++----
 .../s3/S3RetryingInputStream.java             | 29 +++++++++++++++----
 2 files changed, 47 insertions(+), 10 deletions(-)

diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java
index 583d1477fbaa9..085e357da5ae9 100644
--- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java
+++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java
@@ -7,7 +7,6 @@
  */
 package org.elasticsearch.repositories.s3;
 
-import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
@@ -26,6 +25,7 @@
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.indices.recovery.RecoverySettings;
@@ -44,12 +44,14 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
@@ -226,7 +228,6 @@ List<MultipartUpload> listMultipartUploads() {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106457")
     public void testReadFromPositionLargerThanBlobLength() {
         final var blobName = randomIdentifier();
         final var blobBytes = randomBytesReference(randomIntBetween(100, 2_000));
@@ -239,9 +240,26 @@ public void testReadFromPositionLargerThanBlobLength() {
 
         long position = randomLongBetween(blobBytes.length(), Long.MAX_VALUE - 1L);
         long length = randomLongBetween(1L, Long.MAX_VALUE - position);
-        var exception = expectThrows(AmazonClientException.class, () -> readBlob(repository, blobName, position, length));
 
-        assertThat(exception, instanceOf(AmazonS3Exception.class));
-        assertThat(((AmazonS3Exception) exception).getStatusCode(), equalTo(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()));
+        var exception = expectThrows(UncategorizedExecutionException.class, () -> readBlob(repository, blobName, position, length));
+        assertThat(exception.getCause(), instanceOf(ExecutionException.class));
+        assertThat(exception.getCause().getCause(), instanceOf(IOException.class));
+        assertThat(
+            exception.getCause().getCause().getMessage(),
+            containsString(
+                "Requested range [start="
+                    + position
+                    + ", end="
+                    + (position + length - 1L)
+                    + ", currentOffset=0] cannot be satisfied for blob object ["
+                    + repository.basePath().buildAsString()
+                    + blobName
+                    + ']'
+            )
+        );
+        assertThat(
+            asInstanceOf(AmazonS3Exception.class, exception.getRootCause()).getStatusCode(),
+            equalTo(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus())
+        );
     }
 }
diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java
index 998455a658406..d08ff5eefd20f 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.common.blobstore.OperationPurpose;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.repositories.s3.S3BlobStore.Operation;
+import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -94,16 +95,34 @@ private void openStreamWithRetry() throws IOException {
                         : "requesting beyond end, start = " + start + " offset=" + currentOffset + " end=" + end;
                     getObjectRequest.setRange(Math.addExact(start, currentOffset), end);
                 }
-                final S3Object s3Object = SocketAccess.doPrivileged(() -> clientReference.client().getObject(getObjectRequest));
                 this.currentStreamFirstOffset = Math.addExact(start, currentOffset);
+                final S3Object s3Object = SocketAccess.doPrivileged(() -> clientReference.client().getObject(getObjectRequest));
                 this.currentStreamLastOffset = Math.addExact(currentStreamFirstOffset, getStreamLength(s3Object));
                 this.currentStream = s3Object.getObjectContent();
                 return;
             } catch (AmazonClientException e) {
-                if (e instanceof AmazonS3Exception amazonS3Exception && 404 == amazonS3Exception.getStatusCode()) {
-                    throw addSuppressedExceptions(
-                        new NoSuchFileException("Blob object [" + blobKey + "] not found: " + amazonS3Exception.getMessage())
-                    );
+                if (e instanceof AmazonS3Exception amazonS3Exception) {
+                    if (amazonS3Exception.getStatusCode() == RestStatus.NOT_FOUND.getStatus()) {
+                        throw addSuppressedExceptions(
+                            new NoSuchFileException("Blob object [" + blobKey + "] not found: " + amazonS3Exception.getMessage())
+                        );
+                    }
+                    if (amazonS3Exception.getStatusCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()) {
+                        throw addSuppressedExceptions(
+                            new IOException(
+                                "Requested range [start="
+                                    + start
+                                    + ", end="
+                                    + end
+                                    + ", currentOffset="
+                                    + currentOffset
+                                    + "] cannot be satisfied for blob object ["
+                                    + blobKey
+                                    + ']',
+                                amazonS3Exception
+                            )
+                        );
+                    }
                 }
 
                 if (attempt == 1) {

From a8188f8173caf07023aedc490d22fd9050cdab1e Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Fri, 22 Mar 2024 11:12:45 +0000
Subject: [PATCH 14/31] [ML] Fix Array out of bounds exception in the XLM
 Roberta tokenizer (#106655)

Increases the buffer size for the normalised form of the input unicode
character. Certain characters can have surprisingly long normalised forms
---
 docs/changelog/106655.yaml                                   | 5 +++++
 .../nlp/tokenizers/PrecompiledCharMapNormalizer.java         | 5 +----
 .../nlp/tokenizers/PrecompiledCharMapNormalizerTests.java    | 5 +++++
 3 files changed, 11 insertions(+), 4 deletions(-)
 create mode 100644 docs/changelog/106655.yaml

diff --git a/docs/changelog/106655.yaml b/docs/changelog/106655.yaml
new file mode 100644
index 0000000000000..98078595d5f0c
--- /dev/null
+++ b/docs/changelog/106655.yaml
@@ -0,0 +1,5 @@
+pr: 106655
+summary: Fix Array out of bounds exception in the XLM Roberta tokenizer
+area: Machine Learning
+type: bug
+issues: []
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/PrecompiledCharMapNormalizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/PrecompiledCharMapNormalizer.java
index 836c9a78f19d9..93dc8077196d7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/PrecompiledCharMapNormalizer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/PrecompiledCharMapNormalizer.java
@@ -73,10 +73,8 @@ static Config fromBase64EncodedResource(String resourcePath) throws IOException
     private final int[] offsets;
     // The entire normalized bytes representations delimited by NULL
     private final byte[] normalizedStrUtf8Bytes;
-    // Continually reused to copy a single char into utf8 bytes
-    private final byte[] reusableCharByteBuffer = new byte[4];
     // reusable char buffer for decoding utf8 bytes to determine char offset corrections
-    private final char[] reusableCharDecodeBuffer = new char[8];
+    private final char[] reusableCharDecodeBuffer = new char[64];
     private Reader transformedInput;
 
     public PrecompiledCharMapNormalizer(int[] offsets, String normalizedStr, Reader in) {
@@ -172,7 +170,6 @@ Reader normalize(CharSequence str) {
         ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(CharBuffer.wrap(str));
         byte[] strBytes = new byte[byteBuffer.limit()];
         byteBuffer.get(strBytes);
-        int[] strCp = str.codePoints().toArray();
         BreakIterator b = BreakIterator.getCharacterInstance(Locale.ROOT);
         b.setText(str);
         // We iterate the whole string, so b.first() is always `0`
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/PrecompiledCharMapNormalizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/PrecompiledCharMapNormalizerTests.java
index d542b97eee192..eef9902d35e59 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/PrecompiledCharMapNormalizerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/PrecompiledCharMapNormalizerTests.java
@@ -57,6 +57,11 @@ public void testEmoji() throws IOException {
         assertNormalization("😀", parsed, "😀");
     }
 
+    public void testCharThatNormalizesToLongText() throws IOException {
+        PrecompiledCharMapNormalizer.Config parsed = loadTestCharMap();
+        assertNormalization("ﷺ", parsed, "صلى الله عليه وسلم");
+    }
+
     private void assertNormalization(String input, PrecompiledCharMapNormalizer.Config config, String expected) throws IOException {
         PrecompiledCharMapNormalizer normalizer = new PrecompiledCharMapNormalizer(
             config.offsets(),

From c41df745a6e0c954ee8827e7bac7fd0b71e2bb52 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Fri, 22 Mar 2024 12:31:16 +0100
Subject: [PATCH 15/31] Move more XContent parsers that are only used in tests
 to test codebase (#105801)

Just like a couple times before, moving a couple more of the test only
parsers to the test codebase to save code-size etc.
---
 .../geoip/stats/GeoIpDownloaderStats.java     |  34 +----
 .../GeoIpDownloaderStatsSerializingTests.java |  17 ++-
 .../action/DocWriteResponse.java              |   2 +-
 .../node/tasks/list/ListTasksResponse.java    |  41 +----
 .../verify/VerifyRepositoryResponse.java      |  22 ---
 .../ClusterUpdateSettingsResponse.java        |  25 +--
 .../status/SnapshotsStatusResponse.java       |  22 ---
 .../GetScriptContextResponse.java             |  29 +---
 .../analyze/ReloadAnalyzersResponse.java      |  57 +------
 .../admin/indices/open/OpenIndexResponse.java |  16 --
 .../validate/query/ValidateQueryResponse.java |  41 -----
 .../action/delete/DeleteResponse.java         |  20 ---
 .../action/explain/ExplainResponse.java       |  54 +------
 .../action/index/IndexResponse.java           |  20 ---
 .../support/master/AcknowledgedResponse.java  |   2 +-
 .../master/ShardsAcknowledgedResponse.java    |   2 +-
 .../support/tasks/BaseTasksResponse.java      |   4 +-
 .../action/update/UpdateResponse.java         |  31 +---
 .../cluster/health/ClusterIndexHealth.java    |  11 --
 .../index/reindex/BulkByScrollTask.java       | 138 -----------------
 .../search/profile/SearchProfileResults.java  |  81 +---------
 .../verify/VerifyRepositoryResponseTests.java |  23 ++-
 .../ClusterUpdateSettingsResponseTests.java   |  17 ++-
 .../status/SnapshotsStatusResponseTests.java  |  19 ++-
 .../GetScriptContextResponseTests.java        |  26 +++-
 .../analyze/ReloadAnalyzersResponseTests.java |  48 +++++-
 .../indices/open/OpenIndexResponseTests.java  |  15 +-
 .../query/ValidateQueryResponseTests.java     |  36 ++++-
 .../action/bulk/BulkItemResponseTests.java    |   7 +-
 .../action/delete/DeleteResponseTests.java    |  14 +-
 .../action/explain/ExplainResponseTests.java  |  41 ++++-
 .../action/index/IndexResponseTests.java      |  12 +-
 .../action/update/UpdateResponseTests.java    |  29 +++-
 .../health/ClusterIndexHealthTests.java       |   9 +-
 .../reindex/BulkByScrollResponseTests.java    |   2 +-
 ...ulkByScrollTaskStatusOrExceptionTests.java |   2 +-
 .../reindex/BulkByScrollTaskStatusTests.java  | 144 +++++++++++++++++-
 .../profile/SearchProfileResultsTests.java    |   3 +-
 .../tasks/ListTasksResponseTests.java         |  40 ++++-
 .../search/SearchResponseUtils.java           |  78 +++++++++-
 .../core/ilm/ExplainLifecycleResponse.java    |  25 ---
 .../action/PreviewTransformAction.java        |  26 ----
 .../ilm/ExplainLifecycleResponseTests.java    |  22 ++-
 .../PreviewTransformsActionResponseTests.java |  27 +++-
 .../action/PutAnalyticsCollectionAction.java  |  15 --
 ...CollectionResponseBWCSerializingTests.java |   3 +-
 46 files changed, 639 insertions(+), 713 deletions(-)

diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStats.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStats.java
index acfda99ae42fc..64b704a484058 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStats.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStats.java
@@ -13,10 +13,8 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.ingest.geoip.GeoIpDownloader;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -25,26 +23,12 @@ public class GeoIpDownloaderStats implements Task.Status {
 
     public static final GeoIpDownloaderStats EMPTY = new GeoIpDownloaderStats(0, 0, 0, 0, 0, 0);
 
-    public static final ConstructingObjectParser<GeoIpDownloaderStats, Void> PARSER = new ConstructingObjectParser<>(
-        "geoip_downloader_stats",
-        a -> new GeoIpDownloaderStats((int) a[0], (int) a[1], (long) a[2], (int) a[3], (int) a[4], a[5] == null ? 0 : (int) a[5])
-    );
-
-    private static final ParseField SUCCESSFUL_DOWNLOADS = new ParseField("successful_downloads");
-    private static final ParseField FAILED_DOWNLOADS = new ParseField("failed_downloads");
-    private static final ParseField TOTAL_DOWNLOAD_TIME = new ParseField("total_download_time");
-    private static final ParseField DATABASES_COUNT = new ParseField("databases_count");
-    private static final ParseField SKIPPED_DOWNLOADS = new ParseField("skipped_updates");
-    private static final ParseField EXPIRED_DATABASES = new ParseField("expired_databases");
-
-    static {
-        PARSER.declareInt(ConstructingObjectParser.constructorArg(), SUCCESSFUL_DOWNLOADS);
-        PARSER.declareInt(ConstructingObjectParser.constructorArg(), FAILED_DOWNLOADS);
-        PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOTAL_DOWNLOAD_TIME);
-        PARSER.declareInt(ConstructingObjectParser.constructorArg(), DATABASES_COUNT);
-        PARSER.declareInt(ConstructingObjectParser.constructorArg(), SKIPPED_DOWNLOADS);
-        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), EXPIRED_DATABASES);
-    }
+    static final ParseField SUCCESSFUL_DOWNLOADS = new ParseField("successful_downloads");
+    static final ParseField FAILED_DOWNLOADS = new ParseField("failed_downloads");
+    static final ParseField TOTAL_DOWNLOAD_TIME = new ParseField("total_download_time");
+    static final ParseField DATABASES_COUNT = new ParseField("databases_count");
+    static final ParseField SKIPPED_DOWNLOADS = new ParseField("skipped_updates");
+    static final ParseField EXPIRED_DATABASES = new ParseField("expired_databases");
 
     private final int successfulDownloads;
     private final int failedDownloads;
@@ -62,7 +46,7 @@ public GeoIpDownloaderStats(StreamInput in) throws IOException {
         expiredDatabases = in.readVInt();
     }
 
-    private GeoIpDownloaderStats(
+    GeoIpDownloaderStats(
         int successfulDownloads,
         int failedDownloads,
         long totalDownloadTime,
@@ -170,10 +154,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static GeoIpDownloaderStats fromXContent(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(successfulDownloads);
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java
index 68b1ac4b28ff7..69e9cc9b5f5e5 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java
@@ -10,15 +10,30 @@
 
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 
 public class GeoIpDownloaderStatsSerializingTests extends AbstractXContentSerializingTestCase<GeoIpDownloaderStats> {
 
+    private static final ConstructingObjectParser<GeoIpDownloaderStats, Void> PARSER = new ConstructingObjectParser<>(
+        "geoip_downloader_stats",
+        a -> new GeoIpDownloaderStats((int) a[0], (int) a[1], (long) a[2], (int) a[3], (int) a[4], a[5] == null ? 0 : (int) a[5])
+    );
+
+    static {
+        PARSER.declareInt(ConstructingObjectParser.constructorArg(), GeoIpDownloaderStats.SUCCESSFUL_DOWNLOADS);
+        PARSER.declareInt(ConstructingObjectParser.constructorArg(), GeoIpDownloaderStats.FAILED_DOWNLOADS);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), GeoIpDownloaderStats.TOTAL_DOWNLOAD_TIME);
+        PARSER.declareInt(ConstructingObjectParser.constructorArg(), GeoIpDownloaderStats.DATABASES_COUNT);
+        PARSER.declareInt(ConstructingObjectParser.constructorArg(), GeoIpDownloaderStats.SKIPPED_DOWNLOADS);
+        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), GeoIpDownloaderStats.EXPIRED_DATABASES);
+    }
+
     @Override
     protected GeoIpDownloaderStats doParseInstance(XContentParser parser) throws IOException {
-        return GeoIpDownloaderStats.fromXContent(parser);
+        return PARSER.parse(parser, null);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
index 230a8154b64ce..fdef41acb16da 100644
--- a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
@@ -309,7 +309,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
      * {@link DocWriteResponse} objects. It always parses the current token, updates the given parsing context accordingly
      * if needed and then immediately returns.
      */
-    protected static void parseInnerToXContent(XContentParser parser, Builder context) throws IOException {
+    public static void parseInnerToXContent(XContentParser parser, Builder context) throws IOException {
         XContentParser.Token token = parser.currentToken();
         ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
index 7d883ad60b4e7..6d052c242c55c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -23,9 +22,6 @@
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -35,13 +31,11 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
-
 /**
  * Returns the list of tasks currently running on the nodes
  */
 public class ListTasksResponse extends BaseTasksResponse {
-    private static final String TASKS = "tasks";
+    public static final String TASKS = "tasks";
 
     private final List<TaskInfo> tasks;
 
@@ -69,35 +63,6 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(tasks);
     }
 
-    protected static <T> ConstructingObjectParser<T, Void> setupParser(
-        String name,
-        TriFunction<List<TaskInfo>, List<TaskOperationFailure>, List<ElasticsearchException>, T> ctor
-    ) {
-        ConstructingObjectParser<T, Void> parser = new ConstructingObjectParser<>(name, true, constructingObjects -> {
-            int i = 0;
-            @SuppressWarnings("unchecked")
-            List<TaskInfo> tasks = (List<TaskInfo>) constructingObjects[i++];
-            @SuppressWarnings("unchecked")
-            List<TaskOperationFailure> tasksFailures = (List<TaskOperationFailure>) constructingObjects[i++];
-            @SuppressWarnings("unchecked")
-            List<ElasticsearchException> nodeFailures = (List<ElasticsearchException>) constructingObjects[i];
-            return ctor.apply(tasks, tasksFailures, nodeFailures);
-        });
-        parser.declareObjectArray(optionalConstructorArg(), TaskInfo.PARSER, new ParseField(TASKS));
-        parser.declareObjectArray(optionalConstructorArg(), (p, c) -> TaskOperationFailure.fromXContent(p), new ParseField(TASK_FAILURES));
-        parser.declareObjectArray(
-            optionalConstructorArg(),
-            (p, c) -> ElasticsearchException.fromXContent(p),
-            new ParseField(NODE_FAILURES)
-        );
-        return parser;
-    }
-
-    private static final ConstructingObjectParser<ListTasksResponse, Void> PARSER = setupParser(
-        "list_tasks_response",
-        ListTasksResponse::new
-    );
-
     /**
      * Returns the list of tasks by node
      */
@@ -250,10 +215,6 @@ public ChunkedToXContentObject groupedByNone() {
         }));
     }
 
-    public static ListTasksResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
-
     @Override
     public String toString() {
         return Strings.toString(ChunkedToXContent.wrapAsToXContent(groupedByNone()), true, true);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
index 354c67cfb416b..8d48141f9e268 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
@@ -14,11 +14,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xcontent.ObjectParser;
-import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -34,12 +31,6 @@ public class VerifyRepositoryResponse extends ActionResponse implements ToXConte
     static final String NAME = "name";
 
     public static class NodeView implements Writeable, ToXContentObject {
-        private static final ObjectParser.NamedObjectParser<NodeView, Void> PARSER;
-        static {
-            ObjectParser<NodeView, Void> internalParser = new ObjectParser<>(NODES, true, null);
-            internalParser.declareString(NodeView::setName, new ParseField(NAME));
-            PARSER = (p, v, name) -> internalParser.parse(p, new NodeView(name), null);
-        }
 
         final String nodeId;
         String name;
@@ -104,15 +95,6 @@ public int hashCode() {
 
     private List<NodeView> nodes;
 
-    private static final ObjectParser<VerifyRepositoryResponse, Void> PARSER = new ObjectParser<>(
-        VerifyRepositoryResponse.class.getName(),
-        true,
-        VerifyRepositoryResponse::new
-    );
-    static {
-        PARSER.declareNamedObjects(VerifyRepositoryResponse::setNodes, NodeView.PARSER, new ParseField("nodes"));
-    }
-
     public VerifyRepositoryResponse() {}
 
     public VerifyRepositoryResponse(StreamInput in) throws IOException {
@@ -157,10 +139,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static VerifyRepositoryResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
-
     @Override
     public String toString() {
         return Strings.toString(this);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java
index a4ec5222e2392..d99cc0b0ef8df 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java
@@ -12,36 +12,19 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
 /**
  * A response for a cluster update settings action.
  */
 public class ClusterUpdateSettingsResponse extends AcknowledgedResponse {
 
-    private static final ParseField PERSISTENT = new ParseField("persistent");
-    private static final ParseField TRANSIENT = new ParseField("transient");
-
-    private static final ConstructingObjectParser<ClusterUpdateSettingsResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "cluster_update_settings_response",
-        true,
-        args -> {
-            return new ClusterUpdateSettingsResponse((boolean) args[0], (Settings) args[1], (Settings) args[2]);
-        }
-    );
-    static {
-        declareAcknowledgedField(PARSER);
-        PARSER.declareObject(constructorArg(), (p, c) -> Settings.fromXContent(p), TRANSIENT);
-        PARSER.declareObject(constructorArg(), (p, c) -> Settings.fromXContent(p), PERSISTENT);
-    }
+    static final ParseField PERSISTENT = new ParseField("persistent");
+    static final ParseField TRANSIENT = new ParseField("transient");
 
     final Settings transientSettings;
     final Settings persistentSettings;
@@ -83,10 +66,6 @@ protected void addCustomFields(XContentBuilder builder, Params params) throws IO
         builder.endObject();
     }
 
-    public static ClusterUpdateSettingsResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
-
     @Override
     public boolean equals(Object o) {
         if (super.equals(o)) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
index 578800edfb691..941f1c8d30b2c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
@@ -13,18 +13,13 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
 
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
 /**
  * Snapshot status response
  */
@@ -55,23 +50,6 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(snapshots);
     }
 
-    private static final ConstructingObjectParser<SnapshotsStatusResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "snapshots_status_response",
-        true,
-        (Object[] parsedObjects) -> {
-            @SuppressWarnings("unchecked")
-            List<SnapshotStatus> snapshots = (List<SnapshotStatus>) parsedObjects[0];
-            return new SnapshotsStatusResponse(snapshots);
-        }
-    );
-    static {
-        PARSER.declareObjectArray(constructorArg(), SnapshotStatus.PARSER, new ParseField("snapshots"));
-    }
-
-    public static SnapshotsStatusResponse fromXContent(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java
index e7568a0c66a37..73cfeb48b96bc 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponse.java
@@ -13,11 +13,9 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.script.ScriptContextInfo;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -31,28 +29,9 @@
 
 public class GetScriptContextResponse extends ActionResponse implements ToXContentObject {
 
-    private static final ParseField CONTEXTS = new ParseField("contexts");
+    static final ParseField CONTEXTS = new ParseField("contexts");
     final Map<String, ScriptContextInfo> contexts;
 
-    @SuppressWarnings("unchecked")
-    public static final ConstructingObjectParser<GetScriptContextResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "get_script_context",
-        true,
-        (a) -> {
-            Map<String, ScriptContextInfo> contexts = ((List<ScriptContextInfo>) a[0]).stream()
-                .collect(Collectors.toMap(ScriptContextInfo::getName, c -> c));
-            return new GetScriptContextResponse(contexts);
-        }
-    );
-
-    static {
-        PARSER.declareObjectArray(
-            ConstructingObjectParser.constructorArg(),
-            (parser, ctx) -> ScriptContextInfo.PARSER.apply(parser, ctx),
-            CONTEXTS
-        );
-    }
-
     GetScriptContextResponse(StreamInput in) throws IOException {
         super(in);
         int size = in.readInt();
@@ -70,7 +49,7 @@ public class GetScriptContextResponse extends ActionResponse implements ToXConte
     }
 
     // Parser constructor
-    private GetScriptContextResponse(Map<String, ScriptContextInfo> contexts) {
+    GetScriptContextResponse(Map<String, ScriptContextInfo> contexts) {
         this.contexts = Map.copyOf(contexts);
     }
 
@@ -96,10 +75,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static GetScriptContextResponse fromXContent(XContentParser parser) throws IOException {
-        return PARSER.apply(parser, null);
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponse.java
index 4f388804f2340..98b11de5bffc9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponse.java
@@ -9,19 +9,14 @@
 package org.elasticsearch.action.admin.indices.analyze;
 
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -29,8 +24,6 @@
 import java.util.Objects;
 import java.util.Set;
 
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
 /**
  * The response object that will be returned when reloading analyzers
  */
@@ -38,10 +31,10 @@ public class ReloadAnalyzersResponse extends BroadcastResponse {
 
     private final Map<String, ReloadDetails> reloadDetails;
 
-    private static final ParseField RELOAD_DETAILS_FIELD = new ParseField("reload_details");
-    private static final ParseField INDEX_FIELD = new ParseField("index");
-    private static final ParseField RELOADED_ANALYZERS_FIELD = new ParseField("reloaded_analyzers");
-    private static final ParseField RELOADED_NODE_IDS_FIELD = new ParseField("reloaded_node_ids");
+    static final ParseField RELOAD_DETAILS_FIELD = new ParseField("reload_details");
+    static final ParseField INDEX_FIELD = new ParseField("index");
+    static final ParseField RELOADED_ANALYZERS_FIELD = new ParseField("reloaded_analyzers");
+    static final ParseField RELOADED_NODE_IDS_FIELD = new ParseField("reloaded_node_ids");
 
     public ReloadAnalyzersResponse(StreamInput in) throws IOException {
         super(in);
@@ -80,48 +73,6 @@ protected void addCustomXContentFields(XContentBuilder builder, Params params) t
         builder.endArray();
     }
 
-    @SuppressWarnings({ "unchecked" })
-    private static final ConstructingObjectParser<ReloadAnalyzersResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "reload_analyzer",
-        true,
-        arg -> {
-            BaseBroadcastResponse response = (BaseBroadcastResponse) arg[0];
-            List<ReloadDetails> results = (List<ReloadDetails>) arg[1];
-            Map<String, ReloadDetails> reloadedNodeIds = new HashMap<>();
-            for (ReloadDetails result : results) {
-                reloadedNodeIds.put(result.getIndexName(), result);
-            }
-            return new ReloadAnalyzersResponse(
-                response.getTotalShards(),
-                response.getSuccessfulShards(),
-                response.getFailedShards(),
-                Arrays.asList(response.getShardFailures()),
-                reloadedNodeIds
-            );
-        }
-    );
-
-    @SuppressWarnings({ "unchecked" })
-    private static final ConstructingObjectParser<ReloadDetails, Void> ENTRY_PARSER = new ConstructingObjectParser<>(
-        "reload_analyzer.entry",
-        true,
-        arg -> {
-            return new ReloadDetails((String) arg[0], new HashSet<>((List<String>) arg[1]), new HashSet<>((List<String>) arg[2]));
-        }
-    );
-
-    static {
-        declareBroadcastFields(PARSER);
-        PARSER.declareObjectArray(constructorArg(), ENTRY_PARSER, RELOAD_DETAILS_FIELD);
-        ENTRY_PARSER.declareString(constructorArg(), INDEX_FIELD);
-        ENTRY_PARSER.declareStringArray(constructorArg(), RELOADED_ANALYZERS_FIELD);
-        ENTRY_PARSER.declareStringArray(constructorArg(), RELOADED_NODE_IDS_FIELD);
-    }
-
-    public static ReloadAnalyzersResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java
index 39d16e35e61a2..605aab5ab02d9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 
@@ -21,16 +19,6 @@
  */
 public class OpenIndexResponse extends ShardsAcknowledgedResponse {
 
-    private static final ConstructingObjectParser<OpenIndexResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "open_index",
-        true,
-        args -> new OpenIndexResponse((boolean) args[0], (boolean) args[1])
-    );
-
-    static {
-        declareAcknowledgedAndShardsAcknowledgedFields(PARSER);
-    }
-
     public OpenIndexResponse(StreamInput in) throws IOException {
         super(in, true);
     }
@@ -44,8 +32,4 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         writeShardsAcknowledged(out);
     }
-
-    public static OpenIndexResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponse.java
index 6ec0be33e3766..aaa06908f72f0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponse.java
@@ -9,23 +9,14 @@
 package org.elasticsearch.action.admin.indices.validate.query;
 
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
-
 /**
  * The response of the validate action.
  *
@@ -36,38 +27,10 @@ public class ValidateQueryResponse extends BroadcastResponse {
     public static final String VALID_FIELD = "valid";
     public static final String EXPLANATIONS_FIELD = "explanations";
 
-    @SuppressWarnings("unchecked")
-    static final ConstructingObjectParser<ValidateQueryResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "validate_query",
-        true,
-        arg -> {
-            BaseBroadcastResponse response = (BaseBroadcastResponse) arg[0];
-            return new ValidateQueryResponse(
-                (boolean) arg[1],
-                (List<QueryExplanation>) arg[2],
-                response.getTotalShards(),
-                response.getSuccessfulShards(),
-                response.getFailedShards(),
-                Arrays.asList(response.getShardFailures())
-            );
-        }
-    );
-    static {
-        declareBroadcastFields(PARSER);
-        PARSER.declareBoolean(constructorArg(), new ParseField(VALID_FIELD));
-        PARSER.declareObjectArray(optionalConstructorArg(), QueryExplanation.PARSER, new ParseField(EXPLANATIONS_FIELD));
-    }
-
     private final boolean valid;
 
     private final List<QueryExplanation> queryExplanations;
 
-    ValidateQueryResponse(StreamInput in) throws IOException {
-        super(in);
-        valid = in.readBoolean();
-        queryExplanations = in.readCollectionAsList(QueryExplanation::new);
-    }
-
     ValidateQueryResponse(
         boolean valid,
         List<QueryExplanation> queryExplanations,
@@ -115,8 +78,4 @@ protected void addCustomXContentFields(XContentBuilder builder, Params params) t
             builder.endArray();
         }
     }
-
-    public static ValidateQueryResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java
index 9fd9c5fcd791f..47202998d3193 100644
--- a/server/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java
@@ -12,12 +12,9 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 
-import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
-
 /**
  * The response of the delete action.
  *
@@ -64,23 +61,6 @@ public String toString() {
         return builder.append("]").toString();
     }
 
-    public static DeleteResponse fromXContent(XContentParser parser) throws IOException {
-        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
-
-        Builder context = new Builder();
-        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
-            parseXContentFields(parser, context);
-        }
-        return context.build();
-    }
-
-    /**
-     * Parse the current token and update the parsing context appropriately.
-     */
-    public static void parseXContentFields(XContentParser parser, Builder context) throws IOException {
-        DocWriteResponse.parseInnerToXContent(parser, context);
-    }
-
     /**
      * Builder class for {@link DeleteResponse}. This builder is usually used during xcontent parsing to
      * temporarily store the parsed values, then the {@link DocWriteResponse.Builder#build()} method is called to
diff --git a/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java b/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java
index 58c50df47c3ce..9d8ba5f126fd5 100644
--- a/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java
@@ -17,14 +17,11 @@
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Objects;
 
 import static org.elasticsearch.common.lucene.Lucene.readExplanation;
@@ -35,14 +32,14 @@
  */
 public class ExplainResponse extends ActionResponse implements ToXContentObject {
 
-    private static final ParseField _INDEX = new ParseField("_index");
-    private static final ParseField _ID = new ParseField("_id");
+    static final ParseField _INDEX = new ParseField("_index");
+    static final ParseField _ID = new ParseField("_id");
     private static final ParseField MATCHED = new ParseField("matched");
-    private static final ParseField EXPLANATION = new ParseField("explanation");
-    private static final ParseField VALUE = new ParseField("value");
-    private static final ParseField DESCRIPTION = new ParseField("description");
-    private static final ParseField DETAILS = new ParseField("details");
-    private static final ParseField GET = new ParseField("get");
+    static final ParseField EXPLANATION = new ParseField("explanation");
+    static final ParseField VALUE = new ParseField("value");
+    static final ParseField DESCRIPTION = new ParseField("description");
+    static final ParseField DETAILS = new ParseField("details");
+    static final ParseField GET = new ParseField("get");
 
     private final String index;
     private final String id;
@@ -136,43 +133,6 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    private static final ConstructingObjectParser<ExplainResponse, Boolean> PARSER = new ConstructingObjectParser<>(
-        "explain",
-        true,
-        (arg, exists) -> new ExplainResponse((String) arg[0], (String) arg[1], exists, (Explanation) arg[2], (GetResult) arg[3])
-    );
-
-    static {
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), _INDEX);
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), _ID);
-        final ConstructingObjectParser<Explanation, Boolean> explanationParser = getExplanationsParser();
-        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), explanationParser, EXPLANATION);
-        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> GetResult.fromXContentEmbedded(p), GET);
-    }
-
-    @SuppressWarnings("unchecked")
-    private static ConstructingObjectParser<Explanation, Boolean> getExplanationsParser() {
-        final ConstructingObjectParser<Explanation, Boolean> explanationParser = new ConstructingObjectParser<>(
-            "explanation",
-            true,
-            arg -> {
-                if ((float) arg[0] > 0) {
-                    return Explanation.match((float) arg[0], (String) arg[1], (Collection<Explanation>) arg[2]);
-                } else {
-                    return Explanation.noMatch((String) arg[1], (Collection<Explanation>) arg[2]);
-                }
-            }
-        );
-        explanationParser.declareFloat(ConstructingObjectParser.constructorArg(), VALUE);
-        explanationParser.declareString(ConstructingObjectParser.constructorArg(), DESCRIPTION);
-        explanationParser.declareObjectArray(ConstructingObjectParser.constructorArg(), explanationParser, DETAILS);
-        return explanationParser;
-    }
-
-    public static ExplainResponse fromXContent(XContentParser parser, boolean exists) {
-        return PARSER.apply(parser, exists);
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java b/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java
index e3598c33d5951..c547eb7ba8f30 100644
--- a/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java
@@ -17,13 +17,10 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.List;
 
-import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
-
 /**
  * A response of an index operation,
  *
@@ -134,23 +131,6 @@ public String toString() {
         return builder.append("]").toString();
     }
 
-    public static IndexResponse fromXContent(XContentParser parser) throws IOException {
-        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
-
-        Builder context = new Builder();
-        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
-            parseXContentFields(parser, context);
-        }
-        return context.build();
-    }
-
-    /**
-     * Parse the current token and update the parsing context appropriately.
-     */
-    public static void parseXContentFields(XContentParser parser, Builder context) throws IOException {
-        DocWriteResponse.parseInnerToXContent(parser, context);
-    }
-
     /**
      * Builder class for {@link IndexResponse}. This builder is usually used during xcontent parsing to
      * temporarily store the parsed values, then the {@link Builder#build()} method is called to
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
index 107bab447e03e..892db8d4a6d04 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
@@ -40,7 +40,7 @@ public class AcknowledgedResponse extends ActionResponse implements IsAcknowledg
     public static final String ACKNOWLEDGED_KEY = "acknowledged";
     private static final ParseField ACKNOWLEDGED = new ParseField(ACKNOWLEDGED_KEY);
 
-    protected static <T extends AcknowledgedResponse> void declareAcknowledgedField(ConstructingObjectParser<T, Void> objectParser) {
+    public static <T extends AcknowledgedResponse> void declareAcknowledgedField(ConstructingObjectParser<T, Void> objectParser) {
         objectParser.declareField(
             constructorArg(),
             (parser, context) -> parser.booleanValue(),
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java b/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java
index f897d8644d4bb..a00495605dbb5 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java
@@ -24,7 +24,7 @@ public class ShardsAcknowledgedResponse extends AcknowledgedResponse {
 
     protected static final ParseField SHARDS_ACKNOWLEDGED = new ParseField("shards_acknowledged");
 
-    protected static <T extends ShardsAcknowledgedResponse> void declareAcknowledgedAndShardsAcknowledgedFields(
+    public static <T extends ShardsAcknowledgedResponse> void declareAcknowledgedAndShardsAcknowledgedFields(
         ConstructingObjectParser<T, Void> objectParser
     ) {
         declareAcknowledgedField(objectParser);
diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java b/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java
index fe6db0c0c4a4b..3e8290ad4fb4a 100644
--- a/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksResponse.java
@@ -32,8 +32,8 @@
  * Base class for responses of task-related operations
  */
 public class BaseTasksResponse extends ActionResponse {
-    protected static final String TASK_FAILURES = "task_failures";
-    protected static final String NODE_FAILURES = "node_failures";
+    public static final String TASK_FAILURES = "task_failures";
+    public static final String NODE_FAILURES = "node_failures";
 
     private List<TaskOperationFailure> taskFailures;
     private List<ElasticsearchException> nodeFailures;
diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateResponse.java b/server/src/main/java/org/elasticsearch/action/update/UpdateResponse.java
index c6454dd6cedd8..8c9ae3f43d5c4 100644
--- a/server/src/main/java/org/elasticsearch/action/update/UpdateResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/update/UpdateResponse.java
@@ -15,15 +15,12 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 
-import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
-
 public class UpdateResponse extends DocWriteResponse {
 
-    private static final String GET = "get";
+    static final String GET = "get";
 
     private GetResult getResult;
 
@@ -114,32 +111,6 @@ public String toString() {
         return builder.append("]").toString();
     }
 
-    public static UpdateResponse fromXContent(XContentParser parser) throws IOException {
-        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
-
-        Builder context = new Builder();
-        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
-            parseXContentFields(parser, context);
-        }
-        return context.build();
-    }
-
-    /**
-     * Parse the current token and update the parsing context appropriately.
-     */
-    public static void parseXContentFields(XContentParser parser, Builder context) throws IOException {
-        XContentParser.Token token = parser.currentToken();
-        String currentFieldName = parser.currentName();
-
-        if (GET.equals(currentFieldName)) {
-            if (token == XContentParser.Token.START_OBJECT) {
-                context.setGetResult(GetResult.fromXContentEmbedded(parser));
-            }
-        } else {
-            DocWriteResponse.parseInnerToXContent(parser, context);
-        }
-    }
-
     /**
      * Builder class for {@link UpdateResponse}. This builder is usually used during xcontent parsing to
      * temporarily store the parsed values, then the {@link DocWriteResponse.Builder#build()} method is called to
diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java
index 887a0d7d5a751..f236a9eff25a2 100644
--- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java
+++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java
@@ -31,7 +31,6 @@
 import java.util.Objects;
 
 import static java.util.Collections.emptyMap;
-import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
@@ -284,16 +283,6 @@ public static ClusterIndexHealth innerFromXContent(XContentParser parser, String
         return PARSER.apply(parser, index);
     }
 
-    public static ClusterIndexHealth fromXContent(XContentParser parser) throws IOException {
-        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
-        XContentParser.Token token = parser.nextToken();
-        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
-        String index = parser.currentName();
-        ClusterIndexHealth parsed = innerFromXContent(parser, index);
-        ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser);
-        return parsed;
-    }
-
     @Override
     public String toString() {
         return "ClusterIndexHealth{"
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index 150948b4e5822..a483dd5cd48e9 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -20,14 +20,9 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
-import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParseException;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentParser.Token;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -42,9 +37,7 @@
 
 import static java.lang.Math.min;
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.core.TimeValue.timeValueNanos;
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 
 /**
  * Task storing information about a currently running BulkByScroll request.
@@ -380,37 +373,6 @@ public static class Status implements Task.Status, SuccessfullyProcessed {
             FIELDS_SET.add(SLICES_FIELD);
         }
 
-        static final ConstructingObjectParser<Tuple<Long, Long>, Void> RETRIES_PARSER = new ConstructingObjectParser<>(
-            "bulk_by_scroll_task_status_retries",
-            true,
-            a -> new Tuple<>(((Long) a[0]), (Long) a[1])
-        );
-        static {
-            RETRIES_PARSER.declareLong(constructorArg(), new ParseField(RETRIES_BULK_FIELD));
-            RETRIES_PARSER.declareLong(constructorArg(), new ParseField(RETRIES_SEARCH_FIELD));
-        }
-
-        public static void declareFields(ObjectParser<? extends StatusBuilder, Void> parser) {
-            parser.declareInt(StatusBuilder::setSliceId, new ParseField(SLICE_ID_FIELD));
-            parser.declareLong(StatusBuilder::setTotal, new ParseField(TOTAL_FIELD));
-            parser.declareLong(StatusBuilder::setUpdated, new ParseField(UPDATED_FIELD));
-            parser.declareLong(StatusBuilder::setCreated, new ParseField(CREATED_FIELD));
-            parser.declareLong(StatusBuilder::setDeleted, new ParseField(DELETED_FIELD));
-            parser.declareInt(StatusBuilder::setBatches, new ParseField(BATCHES_FIELD));
-            parser.declareLong(StatusBuilder::setVersionConflicts, new ParseField(VERSION_CONFLICTS_FIELD));
-            parser.declareLong(StatusBuilder::setNoops, new ParseField(NOOPS_FIELD));
-            parser.declareObject(StatusBuilder::setRetries, RETRIES_PARSER, new ParseField(RETRIES_FIELD));
-            parser.declareLong(StatusBuilder::setThrottled, new ParseField(THROTTLED_RAW_FIELD));
-            parser.declareFloat(StatusBuilder::setRequestsPerSecond, new ParseField(REQUESTS_PER_SEC_FIELD));
-            parser.declareString(StatusBuilder::setReasonCancelled, new ParseField(CANCELED_FIELD));
-            parser.declareLong(StatusBuilder::setThrottledUntil, new ParseField(THROTTLED_UNTIL_RAW_FIELD));
-            parser.declareObjectArray(
-                StatusBuilder::setSliceStatuses,
-                (p, c) -> StatusOrException.fromXContent(p),
-                new ParseField(SLICES_FIELD)
-            );
-        }
-
         private final Integer sliceId;
         private final long total;
         private final long updated;
@@ -571,11 +533,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             return builder.endObject();
         }
 
-        /**
-         * We need to write a manual parser for this because of {@link StatusOrException}. Since
-         * {@link StatusOrException#fromXContent(XContentParser)} tries to peek at a field first before deciding
-         * what needs to be it cannot use an {@link ObjectParser}.
-         */
         public XContentBuilder innerXContent(XContentBuilder builder, Params params) throws IOException {
             if (sliceId != null) {
                 builder.field(SLICE_ID_FIELD, sliceId);
@@ -617,61 +574,6 @@ public XContentBuilder innerXContent(XContentBuilder builder, Params params) thr
             return builder;
         }
 
-        public static Status fromXContent(XContentParser parser) throws IOException {
-            XContentParser.Token token;
-            if (parser.currentToken() == Token.START_OBJECT) {
-                token = parser.nextToken();
-            } else {
-                token = parser.nextToken();
-            }
-            ensureExpectedToken(Token.START_OBJECT, token, parser);
-            token = parser.nextToken();
-            ensureExpectedToken(Token.FIELD_NAME, token, parser);
-            return innerFromXContent(parser);
-        }
-
-        public static Status innerFromXContent(XContentParser parser) throws IOException {
-            Token token = parser.currentToken();
-            String fieldName = parser.currentName();
-            ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
-            StatusBuilder builder = new StatusBuilder();
-            while ((token = parser.nextToken()) != Token.END_OBJECT) {
-                if (token == Token.FIELD_NAME) {
-                    fieldName = parser.currentName();
-                } else if (token == Token.START_OBJECT) {
-                    if (fieldName.equals(Status.RETRIES_FIELD)) {
-                        builder.setRetries(Status.RETRIES_PARSER.parse(parser, null));
-                    } else {
-                        parser.skipChildren();
-                    }
-                } else if (token == Token.START_ARRAY) {
-                    if (fieldName.equals(Status.SLICES_FIELD)) {
-                        while ((token = parser.nextToken()) != Token.END_ARRAY) {
-                            builder.addToSliceStatuses(StatusOrException.fromXContent(parser));
-                        }
-                    } else {
-                        parser.skipChildren();
-                    }
-                } else { // else if it is a value
-                    switch (fieldName) {
-                        case Status.SLICE_ID_FIELD -> builder.setSliceId(parser.intValue());
-                        case Status.TOTAL_FIELD -> builder.setTotal(parser.longValue());
-                        case Status.UPDATED_FIELD -> builder.setUpdated(parser.longValue());
-                        case Status.CREATED_FIELD -> builder.setCreated(parser.longValue());
-                        case Status.DELETED_FIELD -> builder.setDeleted(parser.longValue());
-                        case Status.BATCHES_FIELD -> builder.setBatches(parser.intValue());
-                        case Status.VERSION_CONFLICTS_FIELD -> builder.setVersionConflicts(parser.longValue());
-                        case Status.NOOPS_FIELD -> builder.setNoops(parser.longValue());
-                        case Status.THROTTLED_RAW_FIELD -> builder.setThrottled(parser.longValue());
-                        case Status.REQUESTS_PER_SEC_FIELD -> builder.setRequestsPerSecond(parser.floatValue());
-                        case Status.CANCELED_FIELD -> builder.setReasonCancelled(parser.text());
-                        case Status.THROTTLED_UNTIL_RAW_FIELD -> builder.setThrottledUntil(parser.longValue());
-                    }
-                }
-            }
-            return builder.buildStatus();
-        }
-
         @Override
         public String toString() {
             StringBuilder builder = new StringBuilder();
@@ -937,46 +839,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             return builder;
         }
 
-        /**
-         * Since {@link StatusOrException} can contain either an {@link Exception} or a {@link Status} we need to peek
-         * at a field first before deciding what needs to be parsed since the same object could contains either.
-         * The {@link #EXPECTED_EXCEPTION_FIELDS} contains the fields that are expected when the serialised object
-         * was an instance of exception and the {@link Status#FIELDS_SET} is the set of fields expected when the
-         * serialized object was an instance of Status.
-         */
-        public static StatusOrException fromXContent(XContentParser parser) throws IOException {
-            XContentParser.Token token = parser.currentToken();
-            if (token == null) {
-                token = parser.nextToken();
-            }
-            if (token == Token.VALUE_NULL) {
-                return null;
-            } else {
-                ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
-                token = parser.nextToken();
-                // This loop is present only to ignore unknown tokens. It breaks as soon as we find a field
-                // that is allowed.
-                while (token != Token.END_OBJECT) {
-                    ensureExpectedToken(Token.FIELD_NAME, token, parser);
-                    String fieldName = parser.currentName();
-                    // weird way to ignore unknown tokens
-                    if (Status.FIELDS_SET.contains(fieldName)) {
-                        return new StatusOrException(Status.innerFromXContent(parser));
-                    } else if (EXPECTED_EXCEPTION_FIELDS.contains(fieldName)) {
-                        return new StatusOrException(ElasticsearchException.innerFromXContent(parser, false));
-                    } else {
-                        // Ignore unknown tokens
-                        token = parser.nextToken();
-                        if (token == Token.START_OBJECT || token == Token.START_ARRAY) {
-                            parser.skipChildren();
-                        }
-                        token = parser.nextToken();
-                    }
-                }
-                throw new XContentParseException("Unable to parse StatusFromException. Expected fields not found.");
-            }
-        }
-
         @Override
         public String toString() {
             if (exception != null) {
diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java
index 1cc6810f8e575..cb15e9af8956a 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java
@@ -15,36 +15,28 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult;
-import org.elasticsearch.search.profile.query.QueryProfileShardResult;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
-
 /**
  * Profile results for all shards.
  */
 public final class SearchProfileResults implements Writeable, ToXContentFragment {
 
     private static final Logger logger = LogManager.getLogger(SearchProfileResults.class);
-    private static final String ID_FIELD = "id";
+    public static final String ID_FIELD = "id";
     private static final String NODE_ID_FIELD = "node_id";
     private static final String CLUSTER_FIELD = "cluster";
     private static final String INDEX_NAME_FIELD = "index";
     private static final String SHARD_ID_FIELD = "shard_id";
-    private static final String SHARDS_FIELD = "shards";
+    public static final String SHARDS_FIELD = "shards";
     public static final String PROFILE_FIELD = "profile";
 
     // map key is the composite "id" of form [nodeId][(clusterName:)indexName][shardId] created from SearchShardTarget.toString
@@ -117,75 +109,6 @@ public String toString() {
         return Strings.toString(this);
     }
 
-    public static SearchProfileResults fromXContent(XContentParser parser) throws IOException {
-        XContentParser.Token token = parser.currentToken();
-        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
-        Map<String, SearchProfileShardResult> profileResults = new HashMap<>();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.START_ARRAY) {
-                if (SHARDS_FIELD.equals(parser.currentName())) {
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        parseProfileResultsEntry(parser, profileResults);
-                    }
-                } else {
-                    parser.skipChildren();
-                }
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                parser.skipChildren();
-            }
-        }
-        return new SearchProfileResults(profileResults);
-    }
-
-    private static void parseProfileResultsEntry(XContentParser parser, Map<String, SearchProfileShardResult> searchProfileResults)
-        throws IOException {
-        XContentParser.Token token = parser.currentToken();
-        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
-        SearchProfileDfsPhaseResult searchProfileDfsPhaseResult = null;
-        List<QueryProfileShardResult> queryProfileResults = new ArrayList<>();
-        AggregationProfileShardResult aggProfileShardResult = null;
-        ProfileResult fetchResult = null;
-        String id = null;
-        String currentFieldName = null;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                if (ID_FIELD.equals(currentFieldName)) {
-                    id = parser.text();
-                } else {
-                    parser.skipChildren();
-                }
-            } else if (token == XContentParser.Token.START_ARRAY) {
-                if ("searches".equals(currentFieldName)) {
-                    while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        queryProfileResults.add(QueryProfileShardResult.fromXContent(parser));
-                    }
-                } else if (AggregationProfileShardResult.AGGREGATIONS.equals(currentFieldName)) {
-                    aggProfileShardResult = AggregationProfileShardResult.fromXContent(parser);
-                } else {
-                    parser.skipChildren();
-                }
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                if ("dfs".equals(currentFieldName)) {
-                    searchProfileDfsPhaseResult = SearchProfileDfsPhaseResult.fromXContent(parser);
-                } else if ("fetch".equals(currentFieldName)) {
-                    fetchResult = ProfileResult.fromXContent(parser);
-                } else {
-                    parser.skipChildren();
-                }
-            } else {
-                parser.skipChildren();
-            }
-        }
-        SearchProfileShardResult result = new SearchProfileShardResult(
-            new SearchProfileQueryPhaseResult(queryProfileResults, aggProfileShardResult),
-            fetchResult
-        );
-        result.getQueryPhase().setSearchProfileDfsPhaseResult(searchProfileDfsPhaseResult);
-        searchProfileResults.put(id, result);
-    }
-
     /**
      * Parsed representation of a composite id used for shards in a profile.
      * The composite id format is specified/created via the {@code SearchShardTarget} method.
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponseTests.java
index 8f5712d90487f..bf90d962912c5 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponseTests.java
@@ -8,6 +8,8 @@
 package org.elasticsearch.action.admin.cluster.repositories.verify;
 
 import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.util.ArrayList;
@@ -15,9 +17,28 @@
 
 public class VerifyRepositoryResponseTests extends AbstractXContentTestCase<VerifyRepositoryResponse> {
 
+    private static final ObjectParser<VerifyRepositoryResponse, Void> PARSER = new ObjectParser<>(
+        VerifyRepositoryResponse.class.getName(),
+        true,
+        VerifyRepositoryResponse::new
+    );
+    static {
+        ObjectParser<VerifyRepositoryResponse.NodeView, Void> internalParser = new ObjectParser<>(
+            VerifyRepositoryResponse.NODES,
+            true,
+            null
+        );
+        internalParser.declareString(VerifyRepositoryResponse.NodeView::setName, new ParseField(VerifyRepositoryResponse.NAME));
+        PARSER.declareNamedObjects(
+            VerifyRepositoryResponse::setNodes,
+            (p, v, name) -> internalParser.parse(p, new VerifyRepositoryResponse.NodeView(name), null),
+            new ParseField("nodes")
+        );
+    }
+
     @Override
     protected VerifyRepositoryResponse doParseInstance(XContentParser parser) {
-        return VerifyRepositoryResponse.fromXContent(parser);
+        return PARSER.apply(parser, null);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java
index 3f5692c30cfef..3d46994faacf7 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java
@@ -14,17 +14,32 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.util.List;
 import java.util.Set;
 import java.util.function.Predicate;
 
+import static org.elasticsearch.action.support.master.AcknowledgedResponse.declareAcknowledgedField;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+
 public class ClusterUpdateSettingsResponseTests extends AbstractXContentSerializingTestCase<ClusterUpdateSettingsResponse> {
 
+    private static final ConstructingObjectParser<ClusterUpdateSettingsResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "cluster_update_settings_response",
+        true,
+        args -> new ClusterUpdateSettingsResponse((boolean) args[0], (Settings) args[1], (Settings) args[2])
+    );
+    static {
+        declareAcknowledgedField(PARSER);
+        PARSER.declareObject(constructorArg(), (p, c) -> Settings.fromXContent(p), ClusterUpdateSettingsResponse.TRANSIENT);
+        PARSER.declareObject(constructorArg(), (p, c) -> Settings.fromXContent(p), ClusterUpdateSettingsResponse.PERSISTENT);
+    }
+
     @Override
     protected ClusterUpdateSettingsResponse doParseInstance(XContentParser parser) {
-        return ClusterUpdateSettingsResponse.fromXContent(parser);
+        return PARSER.apply(parser, null);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponseTests.java
index f90b37f75fa41..21cba892669d0 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponseTests.java
@@ -10,6 +10,8 @@
 
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -17,11 +19,26 @@
 import java.util.List;
 import java.util.function.Predicate;
 
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+
 public class SnapshotsStatusResponseTests extends AbstractChunkedSerializingTestCase<SnapshotsStatusResponse> {
 
+    private static final ConstructingObjectParser<SnapshotsStatusResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "snapshots_status_response",
+        true,
+        (Object[] parsedObjects) -> {
+            @SuppressWarnings("unchecked")
+            List<SnapshotStatus> snapshots = (List<SnapshotStatus>) parsedObjects[0];
+            return new SnapshotsStatusResponse(snapshots);
+        }
+    );
+    static {
+        PARSER.declareObjectArray(constructorArg(), SnapshotStatus.PARSER, new ParseField("snapshots"));
+    }
+
     @Override
     protected SnapshotsStatusResponse doParseInstance(XContentParser parser) throws IOException {
-        return SnapshotsStatusResponse.fromXContent(parser);
+        return PARSER.parse(parser, null);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java
index efb1e61e19fa2..41faaf3517e76 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java
@@ -8,14 +8,38 @@
 package org.elasticsearch.action.admin.cluster.storedscripts;
 
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.script.ScriptContextInfo;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class GetScriptContextResponseTests extends AbstractXContentSerializingTestCase<GetScriptContextResponse> {
 
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<GetScriptContextResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "get_script_context",
+        true,
+        (a) -> {
+            Map<String, ScriptContextInfo> contexts = ((List<ScriptContextInfo>) a[0]).stream()
+                .collect(Collectors.toMap(ScriptContextInfo::getName, c -> c));
+            return new GetScriptContextResponse(contexts);
+        }
+    );
+
+    static {
+        PARSER.declareObjectArray(
+            ConstructingObjectParser.constructorArg(),
+            ScriptContextInfo.PARSER::apply,
+            GetScriptContextResponse.CONTEXTS
+        );
+    }
+
     @Override
     protected GetScriptContextResponse createTestInstance() {
         if (randomBoolean()) {
@@ -31,7 +55,7 @@ protected Writeable.Reader<GetScriptContextResponse> instanceReader() {
 
     @Override
     protected GetScriptContextResponse doParseInstance(XContentParser parser) throws IOException {
-        return GetScriptContextResponse.fromXContent(parser);
+        return PARSER.apply(parser, null);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponseTests.java
index a6524932dd775..f0802e471fc38 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/ReloadAnalyzersResponseTests.java
@@ -8,10 +8,12 @@
 package org.elasticsearch.action.admin.indices.analyze;
 
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.AbstractBroadcastResponseTestCase;
 import org.elasticsearch.test.TransportVersionUtils;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -23,8 +25,52 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.action.support.broadcast.BaseBroadcastResponse.declareBroadcastFields;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+
 public class ReloadAnalyzersResponseTests extends AbstractBroadcastResponseTestCase<ReloadAnalyzersResponse> {
 
+    @SuppressWarnings({ "unchecked" })
+    private static final ConstructingObjectParser<ReloadAnalyzersResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "reload_analyzer",
+        true,
+        arg -> {
+            BaseBroadcastResponse response = (BaseBroadcastResponse) arg[0];
+            List<ReloadAnalyzersResponse.ReloadDetails> results = (List<ReloadAnalyzersResponse.ReloadDetails>) arg[1];
+            Map<String, ReloadAnalyzersResponse.ReloadDetails> reloadedNodeIds = new HashMap<>();
+            for (ReloadAnalyzersResponse.ReloadDetails result : results) {
+                reloadedNodeIds.put(result.getIndexName(), result);
+            }
+            return new ReloadAnalyzersResponse(
+                response.getTotalShards(),
+                response.getSuccessfulShards(),
+                response.getFailedShards(),
+                Arrays.asList(response.getShardFailures()),
+                reloadedNodeIds
+            );
+        }
+    );
+
+    @SuppressWarnings({ "unchecked" })
+    private static final ConstructingObjectParser<ReloadAnalyzersResponse.ReloadDetails, Void> ENTRY_PARSER =
+        new ConstructingObjectParser<>(
+            "reload_analyzer.entry",
+            true,
+            arg -> new ReloadAnalyzersResponse.ReloadDetails(
+                (String) arg[0],
+                new HashSet<>((List<String>) arg[1]),
+                new HashSet<>((List<String>) arg[2])
+            )
+        );
+
+    static {
+        declareBroadcastFields(PARSER);
+        PARSER.declareObjectArray(constructorArg(), ENTRY_PARSER, ReloadAnalyzersResponse.RELOAD_DETAILS_FIELD);
+        ENTRY_PARSER.declareString(constructorArg(), ReloadAnalyzersResponse.INDEX_FIELD);
+        ENTRY_PARSER.declareStringArray(constructorArg(), ReloadAnalyzersResponse.RELOADED_ANALYZERS_FIELD);
+        ENTRY_PARSER.declareStringArray(constructorArg(), ReloadAnalyzersResponse.RELOADED_NODE_IDS_FIELD);
+    }
+
     @Override
     protected ReloadAnalyzersResponse createTestInstance(
         int totalShards,
@@ -50,7 +96,7 @@ public static Map<String, ReloadAnalyzersResponse.ReloadDetails> createRandomRel
 
     @Override
     protected ReloadAnalyzersResponse doParseInstance(XContentParser parser) throws IOException {
-        return ReloadAnalyzersResponse.fromXContent(parser);
+        return PARSER.apply(parser, null);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java
index 962304ef8aadc..a8d8980e6358c 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java
@@ -10,13 +10,26 @@
 
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.XContentParser;
 
+import static org.elasticsearch.action.support.master.ShardsAcknowledgedResponse.declareAcknowledgedAndShardsAcknowledgedFields;
+
 public class OpenIndexResponseTests extends AbstractXContentSerializingTestCase<OpenIndexResponse> {
 
+    private static final ConstructingObjectParser<OpenIndexResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "open_index",
+        true,
+        args -> new OpenIndexResponse((boolean) args[0], (boolean) args[1])
+    );
+
+    static {
+        declareAcknowledgedAndShardsAcknowledgedFields(PARSER);
+    }
+
     @Override
     protected OpenIndexResponse doParseInstance(XContentParser parser) {
-        return OpenIndexResponse.fromXContent(parser);
+        return PARSER.apply(parser, null);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java
index 6bbb2884f1bf3..9ec910e79918c 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java
@@ -10,19 +10,53 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.test.AbstractBroadcastResponseTestCase;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.action.support.broadcast.BaseBroadcastResponse.declareBroadcastFields;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
 public class ValidateQueryResponseTests extends AbstractBroadcastResponseTestCase<ValidateQueryResponse> {
 
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<ValidateQueryResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "validate_query",
+        true,
+        arg -> {
+            BaseBroadcastResponse response = (BaseBroadcastResponse) arg[0];
+            return new ValidateQueryResponse(
+                (boolean) arg[1],
+                (List<QueryExplanation>) arg[2],
+                response.getTotalShards(),
+                response.getSuccessfulShards(),
+                response.getFailedShards(),
+                Arrays.asList(response.getShardFailures())
+            );
+        }
+    );
+    static {
+        declareBroadcastFields(PARSER);
+        PARSER.declareBoolean(constructorArg(), new ParseField(ValidateQueryResponse.VALID_FIELD));
+        PARSER.declareObjectArray(
+            optionalConstructorArg(),
+            QueryExplanation.PARSER,
+            new ParseField(ValidateQueryResponse.EXPLANATIONS_FIELD)
+        );
+    }
+
     private static ValidateQueryResponse createRandomValidateQueryResponse(
         int totalShards,
         int successfulShards,
@@ -60,7 +94,7 @@ private static ValidateQueryResponse createRandomValidateQueryResponse() {
 
     @Override
     protected ValidateQueryResponse doParseInstance(XContentParser parser) throws IOException {
-        return ValidateQueryResponse.fromXContent(parser);
+        return PARSER.apply(parser, null);
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java
index 76b1fa0011540..6c45367baf674 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkItemResponseTests.java
@@ -192,17 +192,16 @@ public static BulkItemResponse itemResponseFromXContent(XContentParser parser, i
         if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
             final IndexResponse.Builder indexResponseBuilder = new IndexResponse.Builder();
             builder = indexResponseBuilder;
-            itemParser = (indexParser) -> IndexResponse.parseXContentFields(indexParser, indexResponseBuilder);
-
+            itemParser = indexParser -> DocWriteResponse.parseInnerToXContent(indexParser, indexResponseBuilder);
         } else if (opType == DocWriteRequest.OpType.UPDATE) {
             final UpdateResponse.Builder updateResponseBuilder = new UpdateResponse.Builder();
             builder = updateResponseBuilder;
-            itemParser = (updateParser) -> UpdateResponse.parseXContentFields(updateParser, updateResponseBuilder);
+            itemParser = updateParser -> UpdateResponseTests.parseXContentFields(updateParser, updateResponseBuilder);
 
         } else if (opType == DocWriteRequest.OpType.DELETE) {
             final DeleteResponse.Builder deleteResponseBuilder = new DeleteResponse.Builder();
             builder = deleteResponseBuilder;
-            itemParser = (deleteParser) -> DeleteResponse.parseXContentFields(deleteParser, deleteResponseBuilder);
+            itemParser = deleteParser -> DocWriteResponse.parseInnerToXContent(deleteParser, deleteResponseBuilder);
         } else {
             throwUnknownField(currentFieldName, parser);
         }
diff --git a/server/src/test/java/org/elasticsearch/action/delete/DeleteResponseTests.java b/server/src/test/java/org/elasticsearch/action/delete/DeleteResponseTests.java
index e7019a583b729..937ac2d26ebb9 100644
--- a/server/src/test/java/org/elasticsearch/action/delete/DeleteResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/delete/DeleteResponseTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.delete;
 
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -26,6 +27,7 @@
 
 import static org.elasticsearch.action.index.IndexResponseTests.assertDocWriteResponse;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_UUID_NA_VALUE;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 
 public class DeleteResponseTests extends ESTestCase {
@@ -102,7 +104,7 @@ private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws
         }
         DeleteResponse parsedDeleteResponse;
         try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
-            parsedDeleteResponse = DeleteResponse.fromXContent(parser);
+            parsedDeleteResponse = parseInstance(parser);
             assertNull(parser.nextToken());
         }
 
@@ -112,6 +114,16 @@ private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws
         assertDocWriteResponse(expectedDeleteResponse, parsedDeleteResponse);
     }
 
+    private static DeleteResponse parseInstance(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
+
+        DeleteResponse.Builder context = new DeleteResponse.Builder();
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            DocWriteResponse.parseInnerToXContent(parser, context);
+        }
+        return context.build();
+    }
+
     /**
      * Returns a tuple of {@link DeleteResponse}s.
      * <p>
diff --git a/server/src/test/java/org/elasticsearch/action/explain/ExplainResponseTests.java b/server/src/test/java/org/elasticsearch/action/explain/ExplainResponseTests.java
index 31fcfe342eb3a..2830d9408e494 100644
--- a/server/src/test/java/org/elasticsearch/action/explain/ExplainResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/explain/ExplainResponseTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.test.RandomObjects;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -24,6 +25,7 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.function.Predicate;
@@ -34,9 +36,46 @@
 
 public class ExplainResponseTests extends AbstractXContentSerializingTestCase<ExplainResponse> {
 
+    private static final ConstructingObjectParser<ExplainResponse, Boolean> PARSER = new ConstructingObjectParser<>(
+        "explain",
+        true,
+        (arg, exists) -> new ExplainResponse((String) arg[0], (String) arg[1], exists, (Explanation) arg[2], (GetResult) arg[3])
+    );
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), ExplainResponse._INDEX);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), ExplainResponse._ID);
+        final ConstructingObjectParser<Explanation, Boolean> explanationParser = getExplanationsParser();
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), explanationParser, ExplainResponse.EXPLANATION);
+        PARSER.declareObject(
+            ConstructingObjectParser.optionalConstructorArg(),
+            (p, c) -> GetResult.fromXContentEmbedded(p),
+            ExplainResponse.GET
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    private static ConstructingObjectParser<Explanation, Boolean> getExplanationsParser() {
+        final ConstructingObjectParser<Explanation, Boolean> explanationParser = new ConstructingObjectParser<>(
+            "explanation",
+            true,
+            arg -> {
+                if ((float) arg[0] > 0) {
+                    return Explanation.match((float) arg[0], (String) arg[1], (Collection<Explanation>) arg[2]);
+                } else {
+                    return Explanation.noMatch((String) arg[1], (Collection<Explanation>) arg[2]);
+                }
+            }
+        );
+        explanationParser.declareFloat(ConstructingObjectParser.constructorArg(), ExplainResponse.VALUE);
+        explanationParser.declareString(ConstructingObjectParser.constructorArg(), ExplainResponse.DESCRIPTION);
+        explanationParser.declareObjectArray(ConstructingObjectParser.constructorArg(), explanationParser, ExplainResponse.DETAILS);
+        return explanationParser;
+    }
+
     @Override
     protected ExplainResponse doParseInstance(XContentParser parser) throws IOException {
-        return ExplainResponse.fromXContent(parser, randomBoolean());
+        return PARSER.apply(parser, randomBoolean());
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/index/IndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/index/IndexResponseTests.java
index ea9e83021e781..c8a8c3853601d 100644
--- a/server/src/test/java/org/elasticsearch/action/index/IndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/index/IndexResponseTests.java
@@ -29,6 +29,7 @@
 
 import static org.elasticsearch.action.support.replication.ReplicationResponseTests.assertShardInfo;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_UUID_NA_VALUE;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 
 public class IndexResponseTests extends ESTestCase {
@@ -111,7 +112,7 @@ private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws
         }
         IndexResponse parsedIndexResponse;
         try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
-            parsedIndexResponse = IndexResponse.fromXContent(parser);
+            parsedIndexResponse = parseInstanceFromXContent(parser);
             assertNull(parser.nextToken());
         }
 
@@ -121,6 +122,15 @@ private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws
         assertDocWriteResponse(expectedIndexResponse, parsedIndexResponse);
     }
 
+    private static IndexResponse parseInstanceFromXContent(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
+        IndexResponse.Builder context = new IndexResponse.Builder();
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            DocWriteResponse.parseInnerToXContent(parser, context);
+        }
+        return context.build();
+    }
+
     public static void assertDocWriteResponse(DocWriteResponse expected, DocWriteResponse actual) {
         assertEquals(expected.getIndex(), actual.getIndex());
         assertEquals(expected.getId(), actual.getId());
diff --git a/server/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java b/server/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java
index 05c974ea9d4d3..d35162287e3ac 100644
--- a/server/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java
@@ -38,6 +38,7 @@
 import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_UUID_NA_VALUE;
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 
@@ -174,7 +175,7 @@ private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws
         }
         UpdateResponse parsedUpdateResponse;
         try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
-            parsedUpdateResponse = UpdateResponse.fromXContent(parser);
+            parsedUpdateResponse = parseInstanceFromXContent(parser);
             assertNull(parser.nextToken());
         }
 
@@ -191,6 +192,32 @@ private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws
         assertToXContentEquivalent(expectedBytes, parsedBytes, xContentType);
     }
 
+    private static UpdateResponse parseInstanceFromXContent(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
+
+        UpdateResponse.Builder context = new UpdateResponse.Builder();
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            parseXContentFields(parser, context);
+        }
+        return context.build();
+    }
+
+    /**
+     * Parse the current token and update the parsing context appropriately.
+     */
+    public static void parseXContentFields(XContentParser parser, UpdateResponse.Builder context) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        String currentFieldName = parser.currentName();
+
+        if (UpdateResponse.GET.equals(currentFieldName)) {
+            if (token == XContentParser.Token.START_OBJECT) {
+                context.setGetResult(GetResult.fromXContentEmbedded(parser));
+            }
+        } else {
+            DocWriteResponse.parseInnerToXContent(parser, context);
+        }
+    }
+
     /**
      * Returns a tuple of {@link UpdateResponse}s.
      * <p>
diff --git a/server/src/test/java/org/elasticsearch/cluster/health/ClusterIndexHealthTests.java b/server/src/test/java/org/elasticsearch/cluster/health/ClusterIndexHealthTests.java
index 9ee942df1c2b0..637a18547b1b2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/health/ClusterIndexHealthTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/health/ClusterIndexHealthTests.java
@@ -26,6 +26,7 @@
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class ClusterIndexHealthTests extends AbstractXContentSerializingTestCase<ClusterIndexHealth> {
@@ -101,7 +102,13 @@ protected Writeable.Reader<ClusterIndexHealth> instanceReader() {
 
     @Override
     protected ClusterIndexHealth doParseInstance(XContentParser parser) throws IOException {
-        return ClusterIndexHealth.fromXContent(parser);
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
+        String index = parser.currentName();
+        ClusterIndexHealth parsed = ClusterIndexHealth.innerFromXContent(parser, index);
+        ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser);
+        return parsed;
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollResponseTests.java b/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollResponseTests.java
index f8162eb987226..48432a0ff4958 100644
--- a/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollResponseTests.java
@@ -51,7 +51,7 @@ public class BulkByScrollResponseTests extends AbstractXContentTestCase<BulkBySc
             new ParseField(BulkByScrollResponse.FAILURES_FIELD)
         );
         // since the result of BulkByScrollResponse.Status are mixed we also parse that in this
-        Status.declareFields(PARSER);
+        BulkByScrollTaskStatusTests.declareFields(PARSER);
     }
 
     private static Object parseFailure(XContentParser parser) throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusOrExceptionTests.java b/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusOrExceptionTests.java
index ccc9f23c09fb1..625963cfcaa3f 100644
--- a/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusOrExceptionTests.java
+++ b/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusOrExceptionTests.java
@@ -41,7 +41,7 @@ static StatusOrException createTestInstanceWithExceptions() {
 
     @Override
     protected StatusOrException doParseInstance(XContentParser parser) throws IOException {
-        return StatusOrException.fromXContent(parser);
+        return BulkByScrollTaskStatusTests.parseStatusOrException(parser);
     }
 
     public static void assertEqualStatusOrException(
diff --git a/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusTests.java b/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusTests.java
index 1d2ec8ae082e8..250d8ad5b19c3 100644
--- a/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusTests.java
+++ b/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusTests.java
@@ -15,9 +15,14 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.reindex.BulkByScrollTask.Status;
 import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentParseException;
 import org.elasticsearch.xcontent.XContentParser;
 import org.hamcrest.Matchers;
 
@@ -32,6 +37,8 @@
 import static java.lang.Math.abs;
 import static java.util.stream.Collectors.toList;
 import static org.apache.lucene.tests.util.TestUtil.randomSimpleString;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.hamcrest.Matchers.equalTo;
 
 public class BulkByScrollTaskStatusTests extends AbstractXContentTestCase<BulkByScrollTask.Status> {
@@ -177,7 +184,142 @@ protected BulkByScrollTask.Status createTestInstance() {
 
     @Override
     protected BulkByScrollTask.Status doParseInstance(XContentParser parser) throws IOException {
-        return BulkByScrollTask.Status.fromXContent(parser);
+        XContentParser.Token token;
+        if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+            token = parser.nextToken();
+        } else {
+            token = parser.nextToken();
+        }
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
+        token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
+        return innerParseStatus(parser);
+    }
+
+    private static final ConstructingObjectParser<Tuple<Long, Long>, Void> RETRIES_PARSER = new ConstructingObjectParser<>(
+        "bulk_by_scroll_task_status_retries",
+        true,
+        a -> new Tuple<>(((Long) a[0]), (Long) a[1])
+    );
+    static {
+        RETRIES_PARSER.declareLong(constructorArg(), new ParseField(BulkByScrollTask.Status.RETRIES_BULK_FIELD));
+        RETRIES_PARSER.declareLong(constructorArg(), new ParseField(BulkByScrollTask.Status.RETRIES_SEARCH_FIELD));
+    }
+
+    public static void declareFields(ObjectParser<? extends BulkByScrollTask.StatusBuilder, Void> parser) {
+        parser.declareInt(BulkByScrollTask.StatusBuilder::setSliceId, new ParseField(BulkByScrollTask.Status.SLICE_ID_FIELD));
+        parser.declareLong(BulkByScrollTask.StatusBuilder::setTotal, new ParseField(BulkByScrollTask.Status.TOTAL_FIELD));
+        parser.declareLong(BulkByScrollTask.StatusBuilder::setUpdated, new ParseField(BulkByScrollTask.Status.UPDATED_FIELD));
+        parser.declareLong(BulkByScrollTask.StatusBuilder::setCreated, new ParseField(BulkByScrollTask.Status.CREATED_FIELD));
+        parser.declareLong(BulkByScrollTask.StatusBuilder::setDeleted, new ParseField(BulkByScrollTask.Status.DELETED_FIELD));
+        parser.declareInt(BulkByScrollTask.StatusBuilder::setBatches, new ParseField(BulkByScrollTask.Status.BATCHES_FIELD));
+        parser.declareLong(
+            BulkByScrollTask.StatusBuilder::setVersionConflicts,
+            new ParseField(BulkByScrollTask.Status.VERSION_CONFLICTS_FIELD)
+        );
+        parser.declareLong(BulkByScrollTask.StatusBuilder::setNoops, new ParseField(BulkByScrollTask.Status.NOOPS_FIELD));
+        parser.declareObject(
+            BulkByScrollTask.StatusBuilder::setRetries,
+            RETRIES_PARSER,
+            new ParseField(BulkByScrollTask.Status.RETRIES_FIELD)
+        );
+        parser.declareLong(BulkByScrollTask.StatusBuilder::setThrottled, new ParseField(BulkByScrollTask.Status.THROTTLED_RAW_FIELD));
+        parser.declareFloat(
+            BulkByScrollTask.StatusBuilder::setRequestsPerSecond,
+            new ParseField(BulkByScrollTask.Status.REQUESTS_PER_SEC_FIELD)
+        );
+        parser.declareString(BulkByScrollTask.StatusBuilder::setReasonCancelled, new ParseField(BulkByScrollTask.Status.CANCELED_FIELD));
+        parser.declareLong(
+            BulkByScrollTask.StatusBuilder::setThrottledUntil,
+            new ParseField(BulkByScrollTask.Status.THROTTLED_UNTIL_RAW_FIELD)
+        );
+        parser.declareObjectArray(
+            BulkByScrollTask.StatusBuilder::setSliceStatuses,
+            (p, c) -> parseStatusOrException(p),
+            new ParseField(BulkByScrollTask.Status.SLICES_FIELD)
+        );
+    }
+
+    private static Status innerParseStatus(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        String fieldName = parser.currentName();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
+        BulkByScrollTask.StatusBuilder builder = new BulkByScrollTask.StatusBuilder();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (fieldName.equals(Status.RETRIES_FIELD)) {
+                    builder.setRetries(RETRIES_PARSER.parse(parser, null));
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (fieldName.equals(Status.SLICES_FIELD)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        builder.addToSliceStatuses(parseStatusOrException(parser));
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            } else { // else if it is a value
+                switch (fieldName) {
+                    case Status.SLICE_ID_FIELD -> builder.setSliceId(parser.intValue());
+                    case Status.TOTAL_FIELD -> builder.setTotal(parser.longValue());
+                    case Status.UPDATED_FIELD -> builder.setUpdated(parser.longValue());
+                    case Status.CREATED_FIELD -> builder.setCreated(parser.longValue());
+                    case Status.DELETED_FIELD -> builder.setDeleted(parser.longValue());
+                    case Status.BATCHES_FIELD -> builder.setBatches(parser.intValue());
+                    case Status.VERSION_CONFLICTS_FIELD -> builder.setVersionConflicts(parser.longValue());
+                    case Status.NOOPS_FIELD -> builder.setNoops(parser.longValue());
+                    case Status.THROTTLED_RAW_FIELD -> builder.setThrottled(parser.longValue());
+                    case Status.REQUESTS_PER_SEC_FIELD -> builder.setRequestsPerSecond(parser.floatValue());
+                    case Status.CANCELED_FIELD -> builder.setReasonCancelled(parser.text());
+                    case Status.THROTTLED_UNTIL_RAW_FIELD -> builder.setThrottledUntil(parser.longValue());
+                }
+            }
+        }
+        return builder.buildStatus();
+    }
+
+    /**
+     * Since {@link BulkByScrollTask.StatusOrException} can contain either an {@link Exception} or a {@link Status} we need to peek
+     * at a field first before deciding what needs to be parsed since the same object could contains either.
+     * The {@link BulkByScrollTask.StatusOrException#EXPECTED_EXCEPTION_FIELDS} contains the fields that are expected when the serialised
+     * object was an instance of exception and the {@link Status#FIELDS_SET} is the set of fields expected when the
+     * serialized object was an instance of Status.
+     */
+    public static BulkByScrollTask.StatusOrException parseStatusOrException(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        if (token == null) {
+            token = parser.nextToken();
+        }
+        if (token == XContentParser.Token.VALUE_NULL) {
+            return null;
+        } else {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
+            token = parser.nextToken();
+            // This loop is present only to ignore unknown tokens. It breaks as soon as we find a field
+            // that is allowed.
+            while (token != XContentParser.Token.END_OBJECT) {
+                ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
+                String fieldName = parser.currentName();
+                // weird way to ignore unknown tokens
+                if (Status.FIELDS_SET.contains(fieldName)) {
+                    return new BulkByScrollTask.StatusOrException(innerParseStatus(parser));
+                } else if (BulkByScrollTask.StatusOrException.EXPECTED_EXCEPTION_FIELDS.contains(fieldName)) {
+                    return new BulkByScrollTask.StatusOrException(ElasticsearchException.innerFromXContent(parser, false));
+                } else {
+                    // Ignore unknown tokens
+                    token = parser.nextToken();
+                    if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY) {
+                        parser.skipChildren();
+                    }
+                    token = parser.nextToken();
+                }
+            }
+            throw new XContentParseException("Unable to parse StatusFromException. Expected fields not found.");
+        }
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java
index f02114a48991c..bda74e75de88c 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchResponseUtils;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
@@ -114,7 +115,7 @@ protected SearchProfileResults doParseInstance(XContentParser parser) throws IOE
         ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
         ensureFieldName(parser, parser.nextToken(), SearchProfileResults.PROFILE_FIELD);
         ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
-        SearchProfileResults result = SearchProfileResults.fromXContent(parser);
+        SearchProfileResults result = SearchResponseUtils.parseSearchProfileResults(parser);
         assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
         assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
         return result;
diff --git a/server/src/test/java/org/elasticsearch/tasks/ListTasksResponseTests.java b/server/src/test/java/org/elasticsearch/tasks/ListTasksResponseTests.java
index f47aaee5ff145..169379441aadd 100644
--- a/server/src/test/java/org/elasticsearch/tasks/ListTasksResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/tasks/ListTasksResponseTests.java
@@ -12,11 +12,15 @@
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -31,12 +35,46 @@
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class ListTasksResponseTests extends AbstractXContentTestCase<ListTasksResponseTests.ListTasksResponseWrapper> {
 
+    private static <T> ConstructingObjectParser<T, Void> setupParser(
+        String name,
+        TriFunction<List<TaskInfo>, List<TaskOperationFailure>, List<ElasticsearchException>, T> ctor
+    ) {
+        ConstructingObjectParser<T, Void> parser = new ConstructingObjectParser<>(name, true, constructingObjects -> {
+            int i = 0;
+            @SuppressWarnings("unchecked")
+            List<TaskInfo> tasks = (List<TaskInfo>) constructingObjects[i++];
+            @SuppressWarnings("unchecked")
+            List<TaskOperationFailure> tasksFailures = (List<TaskOperationFailure>) constructingObjects[i++];
+            @SuppressWarnings("unchecked")
+            List<ElasticsearchException> nodeFailures = (List<ElasticsearchException>) constructingObjects[i];
+            return ctor.apply(tasks, tasksFailures, nodeFailures);
+        });
+        parser.declareObjectArray(optionalConstructorArg(), TaskInfo.PARSER, new ParseField(ListTasksResponse.TASKS));
+        parser.declareObjectArray(
+            optionalConstructorArg(),
+            (p, c) -> TaskOperationFailure.fromXContent(p),
+            new ParseField(BaseTasksResponse.TASK_FAILURES)
+        );
+        parser.declareObjectArray(
+            optionalConstructorArg(),
+            (p, c) -> ElasticsearchException.fromXContent(p),
+            new ParseField(BaseTasksResponse.NODE_FAILURES)
+        );
+        return parser;
+    }
+
+    private static final ConstructingObjectParser<ListTasksResponse, Void> PARSER = setupParser(
+        "list_tasks_response",
+        ListTasksResponse::new
+    );
+
     // ListTasksResponse doesn't directly implement ToXContent because it has multiple XContent representations, so we must wrap here
     public record ListTasksResponseWrapper(ListTasksResponse in) implements ToXContentObject {
         @Override
@@ -108,7 +146,7 @@ private static List<TaskInfo> randomTasks() {
 
     @Override
     protected ListTasksResponseWrapper doParseInstance(XContentParser parser) {
-        return new ListTasksResponseWrapper(ListTasksResponse.fromXContent(parser));
+        return new ListTasksResponseWrapper(PARSER.apply(parser, null));
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java b/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java
index 77b25efd56b35..71837ccf14387 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java
@@ -18,7 +18,13 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.profile.ProfileResult;
+import org.elasticsearch.search.profile.SearchProfileDfsPhaseResult;
+import org.elasticsearch.search.profile.SearchProfileQueryPhaseResult;
 import org.elasticsearch.search.profile.SearchProfileResults;
+import org.elasticsearch.search.profile.SearchProfileShardResult;
+import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult;
+import org.elasticsearch.search.profile.query.QueryProfileShardResult;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -27,6 +33,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -186,7 +193,7 @@ public static SearchResponse parseInnerSearchResponse(XContentParser parser) thr
                 } else if (Suggest.NAME.equals(currentFieldName)) {
                     suggest = Suggest.fromXContent(parser);
                 } else if (SearchProfileResults.PROFILE_FIELD.equals(currentFieldName)) {
-                    profile = SearchProfileResults.fromXContent(parser);
+                    profile = parseSearchProfileResults(parser);
                 } else if (RestActions._SHARDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         if (token == XContentParser.Token.FIELD_NAME) {
@@ -389,4 +396,73 @@ private static SearchResponse.Cluster parseCluster(String clusterAlias, XContent
             timedOut
         );
     }
+
+    public static SearchProfileResults parseSearchProfileResults(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
+        Map<String, SearchProfileShardResult> profileResults = new HashMap<>();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.START_ARRAY) {
+                if (SearchProfileResults.SHARDS_FIELD.equals(parser.currentName())) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        parseProfileResultsEntry(parser, profileResults);
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                parser.skipChildren();
+            }
+        }
+        return new SearchProfileResults(profileResults);
+    }
+
+    private static void parseProfileResultsEntry(XContentParser parser, Map<String, SearchProfileShardResult> searchProfileResults)
+        throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
+        SearchProfileDfsPhaseResult searchProfileDfsPhaseResult = null;
+        List<QueryProfileShardResult> queryProfileResults = new ArrayList<>();
+        AggregationProfileShardResult aggProfileShardResult = null;
+        ProfileResult fetchResult = null;
+        String id = null;
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (SearchProfileResults.ID_FIELD.equals(currentFieldName)) {
+                    id = parser.text();
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if ("searches".equals(currentFieldName)) {
+                    while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        queryProfileResults.add(QueryProfileShardResult.fromXContent(parser));
+                    }
+                } else if (AggregationProfileShardResult.AGGREGATIONS.equals(currentFieldName)) {
+                    aggProfileShardResult = AggregationProfileShardResult.fromXContent(parser);
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if ("dfs".equals(currentFieldName)) {
+                    searchProfileDfsPhaseResult = SearchProfileDfsPhaseResult.fromXContent(parser);
+                } else if ("fetch".equals(currentFieldName)) {
+                    fetchResult = ProfileResult.fromXContent(parser);
+                } else {
+                    parser.skipChildren();
+                }
+            } else {
+                parser.skipChildren();
+            }
+        }
+        SearchProfileShardResult result = new SearchProfileShardResult(
+            new SearchProfileQueryPhaseResult(queryProfileResults, aggProfileShardResult),
+            fetchResult
+        );
+        result.getQueryPhase().setSearchProfileDfsPhaseResult(searchProfileDfsPhaseResult);
+        searchProfileResults.put(id, result);
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleResponse.java
index 749304caf6e20..97c7d6d8cb60d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleResponse.java
@@ -12,18 +12,13 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.function.Function;
-import java.util.stream.Collectors;
 
 /**
  * The response object returned by the Explain Lifecycle API.
@@ -37,26 +32,6 @@ public class ExplainLifecycleResponse extends ActionResponse implements ToXConte
 
     private Map<String, IndexLifecycleExplainResponse> indexResponses;
 
-    @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<ExplainLifecycleResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "explain_lifecycle_response",
-        a -> new ExplainLifecycleResponse(
-            ((List<IndexLifecycleExplainResponse>) a[0]).stream()
-                .collect(Collectors.toMap(IndexLifecycleExplainResponse::getIndex, Function.identity()))
-        )
-    );
-    static {
-        PARSER.declareNamedObjects(
-            ConstructingObjectParser.constructorArg(),
-            (p, c, n) -> IndexLifecycleExplainResponse.PARSER.apply(p, c),
-            INDICES_FIELD
-        );
-    }
-
-    public static ExplainLifecycleResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
-
     public ExplainLifecycleResponse(StreamInput in) throws IOException {
         super(in);
         int size = in.readVInt();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java
index 67c5e22902cf2..f06ba16d9da78 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformAction.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -42,7 +41,6 @@
 import java.util.Objects;
 
 import static org.elasticsearch.core.Strings.format;
-import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class PreviewTransformAction extends ActionType<PreviewTransformAction.Response> {
 
@@ -154,26 +152,6 @@ public static class Response extends ActionResponse implements ToXContentObject
         private final List<Map<String, Object>> docs;
         private final TransformDestIndexSettings generatedDestIndexSettings;
 
-        private static final ConstructingObjectParser<Response, Void> PARSER = new ConstructingObjectParser<>(
-            "data_frame_transform_preview",
-            true,
-            args -> {
-                @SuppressWarnings("unchecked")
-                List<Map<String, Object>> docs = (List<Map<String, Object>>) args[0];
-                TransformDestIndexSettings generatedDestIndex = (TransformDestIndexSettings) args[1];
-
-                return new Response(docs, generatedDestIndex);
-            }
-        );
-        static {
-            PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> p.mapOrdered(), PREVIEW);
-            PARSER.declareObject(
-                optionalConstructorArg(),
-                (p, c) -> TransformDestIndexSettings.fromXContent(p),
-                GENERATED_DEST_INDEX_SETTINGS
-            );
-        }
-
         public Response(List<Map<String, Object>> docs, TransformDestIndexSettings generatedDestIndexSettings) {
             this.docs = docs;
             this.generatedDestIndexSettings = generatedDestIndexSettings;
@@ -237,9 +215,5 @@ public int hashCode() {
         public String toString() {
             return Strings.toString(this, true, true);
         }
-
-        public static Response fromXContent(final XContentParser parser) throws IOException {
-            return PARSER.parse(parser, null);
-        }
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleResponseTests.java
index ab084e66c3ad1..937502281b64d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleResponseTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
@@ -19,10 +20,29 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 public class ExplainLifecycleResponseTests extends AbstractXContentSerializingTestCase<ExplainLifecycleResponse> {
 
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<ExplainLifecycleResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "explain_lifecycle_response",
+        a -> new ExplainLifecycleResponse(
+            ((List<IndexLifecycleExplainResponse>) a[0]).stream()
+                .collect(Collectors.toMap(IndexLifecycleExplainResponse::getIndex, Function.identity()))
+        )
+    );
+    static {
+        PARSER.declareNamedObjects(
+            ConstructingObjectParser.constructorArg(),
+            (p, c, n) -> IndexLifecycleExplainResponse.PARSER.apply(p, c),
+            ExplainLifecycleResponse.INDICES_FIELD
+        );
+    }
+
     @Override
     protected ExplainLifecycleResponse createTestInstance() {
         Map<String, IndexLifecycleExplainResponse> indexResponses = new HashMap<>();
@@ -51,7 +71,7 @@ protected ExplainLifecycleResponse mutateInstance(ExplainLifecycleResponse respo
 
     @Override
     protected ExplainLifecycleResponse doParseInstance(XContentParser parser) throws IOException {
-        return ExplainLifecycleResponse.fromXContent(parser);
+        return PARSER.apply(parser, null);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java
index 9613fc83efd50..9a573818fb111 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java
@@ -9,8 +9,10 @@
 
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction.Response;
+import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings;
 import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettingsTests;
 
 import java.io.IOException;
@@ -18,8 +20,31 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
 public class PreviewTransformsActionResponseTests extends AbstractXContentSerializingTestCase<Response> {
 
+    private static final ConstructingObjectParser<Response, Void> PARSER = new ConstructingObjectParser<>(
+        "data_frame_transform_preview",
+        true,
+        args -> {
+            @SuppressWarnings("unchecked")
+            List<Map<String, Object>> docs = (List<Map<String, Object>>) args[0];
+            TransformDestIndexSettings generatedDestIndex = (TransformDestIndexSettings) args[1];
+
+            return new Response(docs, generatedDestIndex);
+        }
+    );
+
+    static {
+        PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> p.mapOrdered(), PreviewTransformAction.Response.PREVIEW);
+        PARSER.declareObject(
+            optionalConstructorArg(),
+            (p, c) -> TransformDestIndexSettings.fromXContent(p),
+            PreviewTransformAction.Response.GENERATED_DEST_INDEX_SETTINGS
+        );
+    }
+
     public static Response randomPreviewResponse() {
         int size = randomIntBetween(0, 10);
         List<Map<String, Object>> data = new ArrayList<>(size);
@@ -32,7 +57,7 @@ public static Response randomPreviewResponse() {
 
     @Override
     protected Response doParseInstance(XContentParser parser) throws IOException {
-        return Response.fromXContent(parser);
+        return PARSER.parse(parser, null);
     }
 
     @Override
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionAction.java
index 8a544f735b570..659c58d2bd1b8 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionAction.java
@@ -146,20 +146,5 @@ protected void addCustomFields(XContentBuilder builder, Params params) throws IO
             builder.field(COLLECTION_NAME_FIELD.getPreferredName(), name);
         }
 
-        private static final ConstructingObjectParser<Response, String> PARSER = new ConstructingObjectParser<>(
-            "put_analytics_collection_response",
-            false,
-            (p) -> {
-                return new Response((boolean) p[0], (String) p[1]);
-            }
-        );
-        static {
-            PARSER.declareString(constructorArg(), COLLECTION_NAME_FIELD);
-        }
-
-        public static Response fromXContent(String resourceName, XContentParser parser) throws IOException {
-            return new Response(AcknowledgedResponse.fromXContent(parser).isAcknowledged(), resourceName);
-        }
-
     }
 }
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java
index ce6259b40765e..f6a13477acae7 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.application.analytics.action;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
@@ -40,7 +41,7 @@ protected PutAnalyticsCollectionAction.Response mutateInstance(PutAnalyticsColle
 
     @Override
     protected PutAnalyticsCollectionAction.Response doParseInstance(XContentParser parser) throws IOException {
-        return PutAnalyticsCollectionAction.Response.fromXContent(this.name, parser);
+        return new PutAnalyticsCollectionAction.Response(AcknowledgedResponse.fromXContent(parser).isAcknowledged(), this.name);
     }
 
     @Override

From d0c7da6bb7f9ff3373c71d8b9b89ac4d877f1660 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 22 Mar 2024 11:32:56 +0000
Subject: [PATCH 16/31] Improves Error Handling in Chunked Inference API
 (#106634)

Previously, if one input failed within a batch call to the chunked inference API, the entire request was labeled as failed. This update alters this process, now allowing individual errors to be conveyed in the response. This empowers the caller to address failures specifically and understand the root cause, rather than receiving a vague error message like: 'Expected a chunked inference [chunked_text_embedding_service_results] received [error]'.
---
 .../results/ErrorChunkedInferenceResults.java | 97 +++++++++++++++++++
 .../InferenceNamedWriteablesProvider.java     |  8 ++
 .../ElasticsearchInternalService.java         |  4 +
 .../services/elser/ElserInternalService.java  |  4 +
 .../ErrorChunkedInferenceResultsTests.java    | 43 ++++++++
 .../ElasticsearchInternalServiceTests.java    |  9 +-
 .../elser/ElserInternalServiceTests.java      |  8 +-
 7 files changed, 171 insertions(+), 2 deletions(-)
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ErrorChunkedInferenceResultsTests.java

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java
new file mode 100644
index 0000000000000..eef864f2e8992
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.inference.results;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.inference.ChunkedInferenceServiceResults;
+import org.elasticsearch.inference.InferenceResults;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class ErrorChunkedInferenceResults implements ChunkedInferenceServiceResults {
+
+    public static final String NAME = "error_chunked";
+
+    private final Exception exception;
+
+    public ErrorChunkedInferenceResults(Exception exception) {
+        this.exception = Objects.requireNonNull(exception);
+    }
+
+    public ErrorChunkedInferenceResults(StreamInput in) throws IOException {
+        this.exception = in.readException();
+    }
+
+    public Exception getException() {
+        return exception;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeException(exception);
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (object == this) {
+            return true;
+        }
+        if (object == null || getClass() != object.getClass()) {
+            return false;
+        }
+        ErrorChunkedInferenceResults that = (ErrorChunkedInferenceResults) object;
+        // Just compare the message for serialization test purposes
+        return Objects.equals(exception.getMessage(), that.exception.getMessage());
+    }
+
+    @Override
+    public int hashCode() {
+        // Just compare the message for serialization test purposes
+        return Objects.hash(exception.getMessage());
+    }
+
+    @Override
+    public List<? extends InferenceResults> transformToCoordinationFormat() {
+        return null;
+    }
+
+    @Override
+    public List<? extends InferenceResults> transformToLegacyFormat() {
+        return null;
+    }
+
+    @Override
+    public Map<String, Object> asMap() {
+        Map<String, Object> asMap = new LinkedHashMap<>();
+        asMap.put(NAME, exception.getMessage());
+        return asMap;
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(NAME, exception.getMessage());
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
index c38b427200744..80fd98c40516e 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.inference.TaskSettings;
 import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingResults;
+import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
 import org.elasticsearch.xpack.core.inference.results.LegacyTextEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults;
@@ -62,6 +63,13 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         );
 
         // Chunked inference results
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                InferenceServiceResults.class,
+                ErrorChunkedInferenceResults.NAME,
+                ErrorChunkedInferenceResults::new
+            )
+        );
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(
                 InferenceServiceResults.class,
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
index 1aafa340268f3..a07ebe56a9258 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingResults;
+import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
 import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
@@ -36,6 +37,7 @@
 import org.elasticsearch.xpack.core.ml.action.StopTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput;
+import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate;
 import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings;
@@ -382,6 +384,8 @@ private List<ChunkedInferenceServiceResults> translateChunkedResults(List<Infere
         for (var inferenceResult : inferenceResults) {
             if (inferenceResult instanceof org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults mlChunkedResult) {
                 translated.add(ChunkedTextEmbeddingResults.ofMlResult(mlChunkedResult));
+            } else if (inferenceResult instanceof ErrorInferenceResults error) {
+                translated.add(new ErrorChunkedInferenceResults(error.getException()));
             } else {
                 throw new ElasticsearchStatusException(
                     "Expected a chunked inference [{}] received [{}]",
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java
index 8e4918a37b5a4..5069724697818 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults;
+import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction;
 import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
@@ -39,6 +40,7 @@
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput;
 import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults;
+import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -388,6 +390,8 @@ private List<ChunkedInferenceServiceResults> translateChunkedResults(List<Infere
         for (var inferenceResult : inferenceResults) {
             if (inferenceResult instanceof ChunkedTextExpansionResults mlChunkedResult) {
                 translated.add(ChunkedSparseEmbeddingResults.ofMlResult(mlChunkedResult));
+            } else if (inferenceResult instanceof ErrorInferenceResults error) {
+                translated.add(new ErrorChunkedInferenceResults(error.getException()));
             } else {
                 throw new ElasticsearchStatusException(
                     "Expected a chunked inference [{}] received [{}]",
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ErrorChunkedInferenceResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ErrorChunkedInferenceResultsTests.java
new file mode 100644
index 0000000000000..4be00ea9e5822
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ErrorChunkedInferenceResultsTests.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.results;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.ElasticsearchTimeoutException;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
+
+import java.io.IOException;
+
+public class ErrorChunkedInferenceResultsTests extends AbstractWireSerializingTestCase<ErrorChunkedInferenceResults> {
+
+    public static ErrorChunkedInferenceResults createRandomResults() {
+        return new ErrorChunkedInferenceResults(
+            randomBoolean()
+                ? new ElasticsearchTimeoutException(randomAlphaOfLengthBetween(10, 50))
+                : new ElasticsearchStatusException(randomAlphaOfLengthBetween(10, 50), randomFrom(RestStatus.values()))
+        );
+    }
+
+    @Override
+    protected Writeable.Reader<ErrorChunkedInferenceResults> instanceReader() {
+        return ErrorChunkedInferenceResults::new;
+    }
+
+    @Override
+    protected ErrorChunkedInferenceResults createTestInstance() {
+        return createRandomResults();
+    }
+
+    @Override
+    protected ErrorChunkedInferenceResults mutateInstance(ErrorChunkedInferenceResults instance) throws IOException {
+        return new ErrorChunkedInferenceResults(new RuntimeException(randomAlphaOfLengthBetween(10, 50)));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
index 4f0deaceb17da..0757012b234bd 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
@@ -24,8 +24,10 @@
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingResults;
+import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
 import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResultsTests;
+import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
 import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings;
 
 import java.util.ArrayList;
@@ -37,6 +39,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.mockito.ArgumentMatchers.any;
@@ -346,6 +349,7 @@ public void testChunkInfer() {
         var mlTrainedModelResults = new ArrayList<InferenceResults>();
         mlTrainedModelResults.add(ChunkedTextEmbeddingResultsTests.createRandomResults());
         mlTrainedModelResults.add(ChunkedTextEmbeddingResultsTests.createRandomResults());
+        mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom")));
         var response = new InferTrainedModelDeploymentAction.Response(mlTrainedModelResults);
 
         ThreadPool threadpool = new TestThreadPool("test");
@@ -372,7 +376,7 @@ public void testChunkInfer() {
 
         var gotResults = new AtomicBoolean();
         var resultsListener = ActionListener.<List<ChunkedInferenceServiceResults>>wrap(chunkedResponse -> {
-            assertThat(chunkedResponse, hasSize(2));
+            assertThat(chunkedResponse, hasSize(3));
             assertThat(chunkedResponse.get(0), instanceOf(ChunkedTextEmbeddingResults.class));
             var result1 = (ChunkedTextEmbeddingResults) chunkedResponse.get(0);
             assertEquals(
@@ -385,6 +389,9 @@ public void testChunkInfer() {
                 ((org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults) mlTrainedModelResults.get(1)).getChunks(),
                 result2.getChunks()
             );
+            var result3 = (ErrorChunkedInferenceResults) chunkedResponse.get(2);
+            assertThat(result3.getException(), instanceOf(RuntimeException.class));
+            assertThat(result3.getException().getMessage(), containsString("boom"));
             gotResults.set(true);
         }, ESTestCase::fail);
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java
index 6da634afddeb0..f2fd195ab8c5a 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java
@@ -23,8 +23,10 @@
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults;
+import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
 import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResultsTests;
+import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -330,6 +332,7 @@ public void testChunkInfer() {
         var mlTrainedModelResults = new ArrayList<InferenceResults>();
         mlTrainedModelResults.add(ChunkedTextExpansionResultsTests.createRandomResults());
         mlTrainedModelResults.add(ChunkedTextExpansionResultsTests.createRandomResults());
+        mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom")));
         var response = new InferTrainedModelDeploymentAction.Response(mlTrainedModelResults);
 
         ThreadPool threadpool = new TestThreadPool("test");
@@ -357,7 +360,7 @@ public void testChunkInfer() {
 
         var gotResults = new AtomicBoolean();
         var resultsListener = ActionListener.<List<ChunkedInferenceServiceResults>>wrap(chunkedResponse -> {
-            assertThat(chunkedResponse, hasSize(2));
+            assertThat(chunkedResponse, hasSize(3));
             assertThat(chunkedResponse.get(0), instanceOf(ChunkedSparseEmbeddingResults.class));
             var result1 = (ChunkedSparseEmbeddingResults) chunkedResponse.get(0);
             assertEquals(
@@ -370,6 +373,9 @@ public void testChunkInfer() {
                 ((org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults) mlTrainedModelResults.get(1)).getChunks(),
                 result2.getChunkedResults()
             );
+            var result3 = (ErrorChunkedInferenceResults) chunkedResponse.get(2);
+            assertThat(result3.getException(), instanceOf(RuntimeException.class));
+            assertThat(result3.getException().getMessage(), containsString("boom"));
             gotResults.set(true);
         }, ESTestCase::fail);
 

From f3f117a3646cb5ca20c58ec0a7d077ebc664dbc7 Mon Sep 17 00:00:00 2001
From: Liam Thompson <32779855+leemthompo@users.noreply.github.com>
Date: Fri, 22 Mar 2024 12:52:30 +0100
Subject: [PATCH 17/31] [DOCS] Update params for Update Connector Filtering API
 (#106662)

---
 .../apis/update-connector-filtering-api.asciidoc | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
index 3e81f0fda2ce7..04c40ebf9fa4e 100644
--- a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
@@ -55,32 +55,32 @@ Contains the set of rules that are actively used for sync jobs. The `active` obj
     The value to be used in conjunction with the rule for matching the contents of the document's field.
     ** `order` (Required, number) +
     The order in which the rules are applied. The first rule to match has its policy applied.
-    ** `created_at` (Optional, datetime) +
+    ** `created_at` (Required, datetime) +
     The timestamp when the rule was added.
-    ** `updated_at` (Optional, datetime) +
+    ** `updated_at` (Required, datetime) +
     The timestamp when the rule was last edited.
 
-  * `advanced_snippet` (Optional, object) +
+  * `advanced_snippet` (Required, object) +
   Used for {enterprise-search-ref}/sync-rules.html#sync-rules-advanced[advanced filtering] at query time, with the following sub-attributes:
     ** `value` (Required, object) +
     A JSON object passed directly to the connector for advanced filtering.
-    ** `created_at` (Optional, datetime) +
+    ** `created_at` (Required, datetime) +
     The timestamp when this JSON object was created.
-    ** `updated_at` (Optional, datetime) +
+    ** `updated_at` (Required, datetime) +
     The timestamp when this JSON object was last edited.
 
-  * `validation` (Optional, object) +
+  * `validation` (Required, object) +
   Provides validation status for the rules, including:
     ** `state` (Required, string) +
     Indicates the validation state: "edited", "valid", or "invalid".
-    ** `errors` (Optional, object) +
+    ** `errors` (Required, object) +
     Contains details about any validation errors, with sub-attributes:
       *** `ids` (Required, string) +
       The ID(s) of any rules deemed invalid.
       *** `messages` (Required, string) +
       Messages explaining what is invalid about the rules.
 
-- `draft` (Optional, object) +
+- `draft` (Required, object) +
 An object identical in structure to the `active` object, but used for drafting and editing filtering rules before they become active.
 
 

From 16787c2b021e67f2b8e60ddaf8802d45fc1a8459 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <danielmitterdorfer@users.noreply.github.com>
Date: Fri, 22 Mar 2024 13:14:07 +0100
Subject: [PATCH 18/31] [Profiling] Fix spurious test failures (#106660)

With this commit we force-merge indices to workaround
elastic/elasticsearch#106657. We also revert a test mute that was
applied to the wrong test case so all integration tests are executed
again.

Relates #106309
Relates #106657
Closes #106308
---
 .../elasticsearch/xpack/profiling/GetStackTracesActionIT.java  | 1 -
 .../org/elasticsearch/xpack/profiling/ProfilingTestCase.java   | 3 +++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java
index 501d564bbda0d..62b8242e7df86 100644
--- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java
+++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java
@@ -42,7 +42,6 @@ public void testGetStackTracesUnfiltered() throws Exception {
         assertEquals("vmlinux", response.getExecutables().get("lHp5_WAgpLy2alrUVab6HA"));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106308")
     public void testGetStackTracesFromAPMWithMatchNoDownsampling() throws Exception {
         BoolQueryBuilder query = QueryBuilders.boolQuery();
         query.must().add(QueryBuilders.termQuery("transaction.name", "encodeSha1"));
diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/ProfilingTestCase.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/ProfilingTestCase.java
index 383f46c97f02f..58b018a13e096 100644
--- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/ProfilingTestCase.java
+++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/ProfilingTestCase.java
@@ -143,6 +143,9 @@ protected final void doSetupData() throws Exception {
         bulkIndex("data/apm-legacy-test.ndjson");
 
         refresh();
+
+        // temporary workaround for #106657, see also #106308.
+        forceMerge();
     }
 
     @After

From 46b5596767d8f267e0e29cce50673d34993325e9 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 22 Mar 2024 12:47:47 +0000
Subject: [PATCH 19/31] Avoid double-sorting results in
 TransportGetSnapshotsAction (#106644)

Today we call `sortSnapshots` twice, once for the per-repository
results, and then again for the combined results. The first call has no
limit or offset, and the sorting is made redundant by the second call,
so only really serves to filter out snapshots which do not match the
`?after` parameter. Meanwhile the `?after` parameter filtering is
redundant in the second call.

This commit separates these two steps to avoid the redundant sorting in
the first step and the redundant filtering in the second.
---
 .../get/TransportGetSnapshotsAction.java      | 49 ++++++++++---------
 1 file changed, 27 insertions(+), 22 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
index 898adf721be33..6d29c36bdcda1 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
@@ -164,8 +164,7 @@ private class GetSnapshotsOperation {
         @Nullable
         private final String fromSortValue;
         private final int offset;
-        @Nullable
-        private final SnapshotSortKey.After after;
+        private final Predicate<SnapshotInfo> afterPredicate;
         private final int size;
 
         // current state
@@ -210,7 +209,6 @@ private class GetSnapshotsOperation {
             this.order = order;
             this.fromSortValue = fromSortValue;
             this.offset = offset;
-            this.after = after;
             this.size = size;
             this.snapshotsInProgress = snapshotsInProgress;
             this.verbose = verbose;
@@ -219,6 +217,7 @@ private class GetSnapshotsOperation {
             this.snapshotNamePredicate = SnapshotNamePredicate.forSnapshots(ignoreUnavailable, snapshots);
             this.fromSortValuePredicates = SnapshotPredicates.forFromSortValue(fromSortValue, sortBy, order);
             this.slmPolicyPredicate = SlmPolicyPredicate.forPolicies(policies);
+            this.afterPredicate = sortBy.getAfterPredicate(after, order);
 
             this.getSnapshotInfoExecutor = new GetSnapshotInfoExecutor(
                 threadPool.info(ThreadPool.Names.SNAPSHOT_META).getMax(),
@@ -344,20 +343,15 @@ private void loadSnapshotInfos(String repo, @Nullable RepositoryData repositoryD
             } else {
                 assert fromSortValuePredicates.isMatchAll() : "filtering is not supported in non-verbose mode";
                 assert slmPolicyPredicate == SlmPolicyPredicate.MATCH_ALL_POLICIES : "filtering is not supported in non-verbose mode";
-                final var currentSnapshots = snapshotsInProgress.forRepo(repo)
-                    .stream()
-                    .map(entry -> SnapshotInfo.inProgress(entry).basic())
-                    .toList();
-
-                final SnapshotsInRepo snapshotInfos;
-                if (repositoryData != null) {
-                    // want non-current snapshots as well, which are found in the repository data
-                    snapshotInfos = buildSimpleSnapshotInfos(toResolve, repo, repositoryData, currentSnapshots);
-                } else {
-                    // only want current snapshots
-                    snapshotInfos = sortSnapshotsWithNoOffsetOrLimit(currentSnapshots);
-                }
-                listener.onResponse(snapshotInfos);
+
+                listener.onResponse(
+                    buildSimpleSnapshotInfos(
+                        toResolve,
+                        repo,
+                        repositoryData,
+                        snapshotsInProgress.forRepo(repo).stream().map(entry -> SnapshotInfo.inProgress(entry).basic()).toList()
+                    )
+                );
             }
         }
 
@@ -446,7 +440,7 @@ public void onFailure(Exception e) {
 
                 .addListener(listener.safeMap(v ->
                 // no need to synchronize access to snapshots: Repository#getSnapshotInfo fails fast but we're on the success path here
-                sortSnapshotsWithNoOffsetOrLimit(snapshots)), executor, threadPool.getThreadContext());
+                applyAfterPredicate(snapshots)), executor, threadPool.getThreadContext());
         }
 
         private SnapshotsInRepo buildSimpleSnapshotInfos(
@@ -455,6 +449,11 @@ private SnapshotsInRepo buildSimpleSnapshotInfos(
             final RepositoryData repositoryData,
             final List<SnapshotInfo> currentSnapshots
         ) {
+            if (repositoryData == null) {
+                // only want current snapshots
+                return applyAfterPredicate(currentSnapshots);
+            } // else want non-current snapshots as well, which are found in the repository data
+
             List<SnapshotInfo> snapshotInfos = new ArrayList<>();
             for (SnapshotInfo snapshotInfo : currentSnapshots) {
                 assert snapshotInfo.startTime() == 0L && snapshotInfo.endTime() == 0L && snapshotInfo.totalShards() == 0L : snapshotInfo;
@@ -483,16 +482,16 @@ private SnapshotsInRepo buildSimpleSnapshotInfos(
                     )
                 );
             }
-            return sortSnapshotsWithNoOffsetOrLimit(snapshotInfos);
+            return applyAfterPredicate(snapshotInfos);
         }
 
-        private SnapshotsInRepo sortSnapshotsWithNoOffsetOrLimit(List<SnapshotInfo> snapshotInfos) {
-            return sortSnapshots(snapshotInfos.stream(), snapshotInfos.size(), 0, GetSnapshotsRequest.NO_LIMIT);
+        private SnapshotsInRepo applyAfterPredicate(List<SnapshotInfo> snapshotInfos) {
+            return new SnapshotsInRepo(snapshotInfos.stream().filter(afterPredicate).toList(), snapshotInfos.size(), 0);
         }
 
         private SnapshotsInRepo sortSnapshots(Stream<SnapshotInfo> snapshotInfoStream, int totalCount, int offset, int size) {
             assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT);
-            final var resultsStream = snapshotInfoStream.filter(sortBy.getAfterPredicate(after, order))
+            final var resultsStream = snapshotInfoStream.peek(this::assertSatisfiesAllPredicates)
                 .sorted(sortBy.getSnapshotInfoComparator(order))
                 .skip(offset);
             if (size == GetSnapshotsRequest.NO_LIMIT) {
@@ -513,6 +512,12 @@ private SnapshotsInRepo sortSnapshots(Stream<SnapshotInfo> snapshotInfoStream, i
             }
         }
 
+        private void assertSatisfiesAllPredicates(SnapshotInfo snapshotInfo) {
+            assert matchesPredicates(snapshotInfo);
+            assert afterPredicate.test(snapshotInfo);
+            assert indices || snapshotInfo.indices().isEmpty();
+        }
+
         private boolean matchesPredicates(SnapshotId snapshotId, RepositoryData repositoryData) {
             if (fromSortValuePredicates.test(snapshotId, repositoryData) == false) {
                 return false;

From 391f010f9b5b111efbe649602733dedf8555a182 Mon Sep 17 00:00:00 2001
From: Luigi Dell'Aquila <luigi.dellaquila@gmail.com>
Date: Fri, 22 Mar 2024 14:01:31 +0100
Subject: [PATCH 20/31] ES|QL: Fix usage of IN operator with TEXT fields
 (#106654)

---
 docs/changelog/106654.yaml                    |  6 ++
 .../predicate/operator/comparison/In.java     |  4 +-
 .../xpack/esql/analysis/AnalyzerTests.java    | 26 +++++++
 .../rest-api-spec/test/esql/80_text.yml       | 76 +++++++++++++++++++
 4 files changed, 110 insertions(+), 2 deletions(-)
 create mode 100644 docs/changelog/106654.yaml

diff --git a/docs/changelog/106654.yaml b/docs/changelog/106654.yaml
new file mode 100644
index 0000000000000..3443b68482443
--- /dev/null
+++ b/docs/changelog/106654.yaml
@@ -0,0 +1,6 @@
+pr: 106654
+summary: "ES|QL: Fix usage of IN operator with TEXT fields"
+area: ES|QL
+type: bug
+issues:
+ - 105379
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
index b20160ac936d6..ab2f9079b610c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
@@ -7,10 +7,10 @@
 
 package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
 
+import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.Expressions;
-import org.elasticsearch.xpack.ql.expression.TypeResolutions;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.InProcessor;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
 import org.elasticsearch.xpack.ql.tree.Source;
@@ -67,7 +67,7 @@ protected boolean areCompatible(DataType left, DataType right) {
 
     @Override
     protected TypeResolution resolveType() { // TODO: move the foldability check from QL's In to SQL's and remove this method
-        TypeResolution resolution = TypeResolutions.isExact(value(), functionName(), DEFAULT);
+        TypeResolution resolution = EsqlTypeResolutions.isExact(value(), functionName(), DEFAULT);
         if (resolution.unresolved()) {
             return resolution;
         }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index 975b31b967fe0..543e7c93526d2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -1775,6 +1775,32 @@ public void testUnsupportedTypesInStats() {
         );
     }
 
+    public void testInOnText() {
+        assertProjectionWithMapping("""
+            from a_index
+            | eval text in (\"a\", \"b\", \"c\")
+            | keep text
+            """, "mapping-multi-field-variation.json", "text");
+
+        assertProjectionWithMapping("""
+            from a_index
+            | eval text in (\"a\", \"b\", \"c\", text)
+            | keep text
+            """, "mapping-multi-field-variation.json", "text");
+
+        assertProjectionWithMapping("""
+            from a_index
+            | eval text not in (\"a\", \"b\", \"c\")
+            | keep text
+            """, "mapping-multi-field-variation.json", "text");
+
+        assertProjectionWithMapping("""
+            from a_index
+            | eval text not in (\"a\", \"b\", \"c\", text)
+            | keep text
+            """, "mapping-multi-field-variation.json", "text");
+    }
+
     private void verifyUnsupported(String query, String errorMessage) {
         verifyUnsupported(query, errorMessage, "mapping-multi-field-variation.json");
     }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml
index 09462691688bf..d73efe1788ce3 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml
@@ -121,6 +121,82 @@ setup:
   - length: { values: 1 }
   - match: { values.0: [ 20, "John", "Payroll Specialist", "baz"] }
 
+---
+"IN on text":
+  - skip:
+      version: " - 8.13.99"
+      reason: "IN on text fixed in v 8.14"
+      features: allowed_warnings_regex
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'from test | where tag IN ("abc", "baz") | keep emp_no, name, job, tag'
+
+  - match: { columns.0.name: "emp_no" }
+  - match: { columns.0.type: "long" }
+  - match: { columns.1.name: "name" }
+  - match: { columns.1.type: "keyword" }
+  - match: { columns.2.name: "job" }
+  - match: { columns.2.type: "text" }
+  - match: { columns.3.name: "tag" }
+  - match: { columns.3.type: "text" }
+
+  - length: { values: 1 }
+  - match: { values.0: [ 20, "John", "Payroll Specialist", "baz"] }
+
+---
+"IN on text and itself":
+  - skip:
+      version: " - 8.13.99"
+      reason: "IN on text fixed in v 8.14"
+      features: allowed_warnings_regex
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'from test | where tag IN ("abc", tag) | keep emp_no, name, job, tag | sort emp_no'
+
+  - match: { columns.0.name: "emp_no" }
+  - match: { columns.0.type: "long" }
+  - match: { columns.1.name: "name" }
+  - match: { columns.1.type: "keyword" }
+  - match: { columns.2.name: "job" }
+  - match: { columns.2.type: "text" }
+  - match: { columns.3.name: "tag" }
+  - match: { columns.3.type: "text" }
+
+  - length: { values: 2 }
+  - match: { values.0: [ 10, "Jenny", "IT Director", "foo bar"] }
+  - match: { values.1: [ 20, "John", "Payroll Specialist", "baz"] }
+
+---
+"NOT IN on text":
+  - skip:
+      version: " - 8.13.99"
+      reason: "IN on text fixed in v 8.14"
+      features: allowed_warnings_regex
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'from test | where tag NOT IN ("abc", "baz") | keep emp_no, name, job, tag'
+
+  - match: { columns.0.name: "emp_no" }
+  - match: { columns.0.type: "long" }
+  - match: { columns.1.name: "name" }
+  - match: { columns.1.type: "keyword" }
+  - match: { columns.2.name: "job" }
+  - match: { columns.2.type: "text" }
+  - match: { columns.3.name: "tag" }
+  - match: { columns.3.type: "text" }
+
+  - length: { values: 1 }
+  - match: { values.0: [ 10, "Jenny", "IT Director", "foo bar"] }
+
 ---
 "eval and filter text":
   - do:

From bd6f3c7f78c7577a8fabd945f3cefb6f11ed28be Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Fri, 22 Mar 2024 09:18:43 -0400
Subject: [PATCH 21/31] Test mute for issue #106647 (#106670)

---
 .../search/query/PartialHitCountCollectorTests.java              | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/test/java/org/elasticsearch/search/query/PartialHitCountCollectorTests.java b/server/src/test/java/org/elasticsearch/search/query/PartialHitCountCollectorTests.java
index c7967f0de5411..c63e2499147c1 100644
--- a/server/src/test/java/org/elasticsearch/search/query/PartialHitCountCollectorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/PartialHitCountCollectorTests.java
@@ -118,6 +118,7 @@ public void testHitCountFromWeightDoesNotEarlyTerminate() throws IOException {
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106647")
     public void testCollectedHitCount() throws Exception {
         Query query = new NonCountingTermQuery(new Term("string", "a1"));
         int threshold = randomIntBetween(1, 10000);

From f02adacceae8d44665f781b0c78331ac80140f62 Mon Sep 17 00:00:00 2001
From: Andrei Dan <andrei.dan@elastic.co>
Date: Fri, 22 Mar 2024 13:38:21 +0000
Subject: [PATCH 22/31] Fix testDataStreamLifecycleDownsampleRollingRestart
 (#106664)

This removes a redundant thread creation when triggering a rolling restart
as the method is already async and drops the check for cluster health as that
might hit a node that's being shut down (the master node in particular)

The test assertions still hold i.e. the successful downsampling of the
source index
---
 ...StreamLifecycleDownsampleDisruptionIT.java | 84 +------------------
 1 file changed, 4 insertions(+), 80 deletions(-)

diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java
index 65a4d84e921a2..afa2e95e1284c 100644
--- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java
+++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java
@@ -31,9 +31,7 @@
 
 import java.util.Collection;
 import java.util.List;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
 
 import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.getBackingIndices;
 import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.putTSDBIndexTemplate;
@@ -57,11 +55,10 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         return settings.build();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/105577")
     @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging")
     public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception {
         final InternalTestCluster cluster = internalCluster();
-        final List<String> masterNodes = cluster.startMasterOnlyNodes(1);
+        cluster.startMasterOnlyNodes(1);
         cluster.startDataOnlyNodes(3);
         ensureStableCluster(cluster.size());
         ensureGreen();
@@ -99,36 +96,12 @@ public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception {
         long sleepTime = randomLongBetween(3000, 4500);
         logger.info("-> giving data stream lifecycle [{}] millis to make some progress before starting the disruption", sleepTime);
         Thread.sleep(sleepTime);
-        final CountDownLatch disruptionStart = new CountDownLatch(1);
-        final CountDownLatch disruptionEnd = new CountDownLatch(1);
         List<String> backingIndices = getBackingIndices(client(), dataStreamName);
         // first generation index
         String sourceIndex = backingIndices.get(0);
-        new Thread(new Disruptor(cluster, sourceIndex, new DisruptionListener() {
-            @Override
-            public void disruptionStart() {
-                disruptionStart.countDown();
-            }
 
-            @Override
-            public void disruptionEnd() {
-                disruptionEnd.countDown();
-            }
-        }, masterNodes.get(0), (ignored) -> {
-            try {
-                cluster.rollingRestart(new InternalTestCluster.RestartCallback() {
-                    @Override
-                    public boolean validateClusterForming() {
-                        return true;
-                    }
-                });
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        })).start();
-
-        waitUntil(() -> getClusterPendingTasks(cluster.client()).pendingTasks().isEmpty(), 60, TimeUnit.SECONDS);
-        ensureStableCluster(cluster.numDataAndMasterNodes());
+        internalCluster().rollingRestart(new InternalTestCluster.RestartCallback() {
+        });
 
         // if the source index has already been downsampled and moved into the data stream just use its name directly
         final String targetIndex = sourceIndex.startsWith("downsample-5m-") ? sourceIndex : "downsample-5m-" + sourceIndex;
@@ -147,55 +120,6 @@ public boolean validateClusterForming() {
                 throw new AssertionError(e);
             }
         }, 60, TimeUnit.SECONDS);
-    }
-
-    interface DisruptionListener {
-        void disruptionStart();
-
-        void disruptionEnd();
-    }
-
-    private class Disruptor implements Runnable {
-        final InternalTestCluster cluster;
-        private final String sourceIndex;
-        private final DisruptionListener listener;
-        private final String clientNode;
-        private final Consumer<String> disruption;
-
-        private Disruptor(
-            final InternalTestCluster cluster,
-            final String sourceIndex,
-            final DisruptionListener listener,
-            final String clientNode,
-            final Consumer<String> disruption
-        ) {
-            this.cluster = cluster;
-            this.sourceIndex = sourceIndex;
-            this.listener = listener;
-            this.clientNode = clientNode;
-            this.disruption = disruption;
-        }
-
-        @Override
-        public void run() {
-            listener.disruptionStart();
-            try {
-                final String candidateNode = cluster.client(clientNode)
-                    .admin()
-                    .cluster()
-                    .prepareSearchShards(sourceIndex)
-                    .get()
-                    .getNodes()[0].getName();
-                logger.info("Candidate node [" + candidateNode + "]");
-                disruption.accept(candidateNode);
-                ensureGreen(sourceIndex);
-                ensureStableCluster(cluster.numDataAndMasterNodes(), clientNode);
-
-            } catch (Exception e) {
-                logger.error("Ignoring Error while injecting disruption [" + e.getMessage() + "]");
-            } finally {
-                listener.disruptionEnd();
-            }
-        }
+        ensureGreen(targetIndex);
     }
 }

From 9a907704b7300f3364d74047a92e79a265952c6c Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 22 Mar 2024 13:46:46 +0000
Subject: [PATCH 23/31] Move `XContent` -> `SnapshotInfo` parsing out of prod
 (#106669)

The code to parse a `SnapshotInfo` object out of an `XContent` response
body is only used in tests, so this commit moves it out of the
production codebase and into the test framework.
---
 .../http/snapshots/RestGetSnapshotsIT.java    |   3 +-
 .../create/CreateSnapshotResponse.java        |  30 +-
 .../create/TransportCreateSnapshotAction.java |   3 +-
 .../elasticsearch/snapshots/SnapshotInfo.java | 258 ++----------------
 .../create/CreateSnapshotResponseTests.java   |   3 +-
 .../snapshots/SnapshotInfoUtils.java          | 244 +++++++++++++++++
 .../xpack/slm/SnapshotLifecycleTaskTests.java |   5 +-
 7 files changed, 285 insertions(+), 261 deletions(-)
 create mode 100644 test/framework/src/main/java/org/elasticsearch/snapshots/SnapshotInfoUtils.java

diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java
index 88d910b61fa52..b12a70ccb8425 100644
--- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java
+++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.elasticsearch.snapshots.SnapshotInfo;
+import org.elasticsearch.snapshots.SnapshotInfoUtils;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -524,7 +525,7 @@ private static GetSnapshotsResponse sortedWithLimit(
     static {
         GET_SNAPSHOT_PARSER.declareObjectArray(
             ConstructingObjectParser.constructorArg(),
-            (p, c) -> SnapshotInfo.SNAPSHOT_INFO_PARSER.apply(p, c).build(),
+            (p, c) -> SnapshotInfoUtils.snapshotInfoFromXContent(p),
             new ParseField("snapshots")
         );
         GET_SNAPSHOT_PARSER.declareObject(
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
index 7062efd301991..4e04506d03d6a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
@@ -14,12 +14,8 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.SnapshotInfo;
-import org.elasticsearch.snapshots.SnapshotInfo.SnapshotInfoBuilder;
-import org.elasticsearch.xcontent.ObjectParser;
-import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -29,24 +25,8 @@
  */
 public class CreateSnapshotResponse extends ActionResponse implements ToXContentObject {
 
-    private static final ObjectParser<CreateSnapshotResponse, Void> PARSER = new ObjectParser<>(
-        CreateSnapshotResponse.class.getName(),
-        true,
-        CreateSnapshotResponse::new
-    );
-
-    static {
-        PARSER.declareObject(
-            CreateSnapshotResponse::setSnapshotInfoFromBuilder,
-            SnapshotInfo.SNAPSHOT_INFO_PARSER,
-            new ParseField("snapshot")
-        );
-    }
-
     @Nullable
-    private SnapshotInfo snapshotInfo;
-
-    CreateSnapshotResponse() {}
+    private final SnapshotInfo snapshotInfo;
 
     public CreateSnapshotResponse(@Nullable SnapshotInfo snapshotInfo) {
         this.snapshotInfo = snapshotInfo;
@@ -57,10 +37,6 @@ public CreateSnapshotResponse(StreamInput in) throws IOException {
         snapshotInfo = in.readOptionalWriteable(SnapshotInfo::readFrom);
     }
 
-    private void setSnapshotInfoFromBuilder(SnapshotInfoBuilder snapshotInfoBuilder) {
-        this.snapshotInfo = snapshotInfoBuilder.build();
-    }
-
     /**
      * Returns snapshot information if snapshot was completed by the time this method returned or null otherwise.
      *
@@ -103,10 +79,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static CreateSnapshotResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
-
     @Override
     public String toString() {
         return "CreateSnapshotResponse{" + "snapshotInfo=" + snapshotInfo + '}';
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
index 8d776b7ae6ecb..02592b722c9e0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -68,7 +69,7 @@ protected void masterOperation(
         if (request.waitForCompletion()) {
             snapshotsService.executeSnapshot(request, listener.map(CreateSnapshotResponse::new));
         } else {
-            snapshotsService.createSnapshot(request, listener.map(snapshot -> new CreateSnapshotResponse()));
+            snapshotsService.createSnapshot(request, listener.map(snapshot -> new CreateSnapshotResponse((SnapshotInfo) null)));
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index 243df88cfab00..8a1f68c867943 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.repositories.RepositoryShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentFragment;
@@ -53,237 +52,40 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContentF
     public static final String INCLUDE_REPOSITORY_XCONTENT_PARAM = "include_repository";
 
     private static final DateFormatter DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time");
-    private static final String SNAPSHOT = "snapshot";
-    private static final String UUID = "uuid";
-    private static final String REPOSITORY = "repository";
-    private static final String INDICES = "indices";
-    private static final String DATA_STREAMS = "data_streams";
-    private static final String STATE = "state";
-    private static final String REASON = "reason";
-    private static final String START_TIME = "start_time";
-    private static final String START_TIME_IN_MILLIS = "start_time_in_millis";
-    private static final String END_TIME = "end_time";
-    private static final String END_TIME_IN_MILLIS = "end_time_in_millis";
-    private static final String DURATION = "duration";
-    private static final String DURATION_IN_MILLIS = "duration_in_millis";
-    private static final String FAILURES = "failures";
-    private static final String SHARDS = "shards";
-    private static final String TOTAL = "total";
-    private static final String FAILED = "failed";
-    private static final String SUCCESSFUL = "successful";
-    private static final String VERSION_ID = "version_id";
-    private static final String VERSION = "version";
-    private static final String NAME = "name";
-    private static final String TOTAL_SHARDS = "total_shards";
-    private static final String SUCCESSFUL_SHARDS = "successful_shards";
-    private static final String INCLUDE_GLOBAL_STATE = "include_global_state";
-    private static final String USER_METADATA = "metadata";
-    private static final String FEATURE_STATES = "feature_states";
-    private static final String INDEX_DETAILS = "index_details";
-
-    private static final String UNKNOWN_REPO_NAME = "_na_";
+
+    static final String SNAPSHOT = "snapshot";
+    static final String UUID = "uuid";
+    static final String REPOSITORY = "repository";
+    static final String INDICES = "indices";
+    static final String DATA_STREAMS = "data_streams";
+    static final String STATE = "state";
+    static final String REASON = "reason";
+    static final String START_TIME = "start_time";
+    static final String START_TIME_IN_MILLIS = "start_time_in_millis";
+    static final String END_TIME = "end_time";
+    static final String END_TIME_IN_MILLIS = "end_time_in_millis";
+    static final String DURATION = "duration";
+    static final String DURATION_IN_MILLIS = "duration_in_millis";
+    static final String FAILURES = "failures";
+    static final String SHARDS = "shards";
+    static final String TOTAL = "total";
+    static final String FAILED = "failed";
+    static final String SUCCESSFUL = "successful";
+    static final String VERSION_ID = "version_id";
+    static final String VERSION = "version";
+    static final String NAME = "name";
+    static final String TOTAL_SHARDS = "total_shards";
+    static final String SUCCESSFUL_SHARDS = "successful_shards";
+    static final String INCLUDE_GLOBAL_STATE = "include_global_state";
+    static final String USER_METADATA = "metadata";
+    static final String FEATURE_STATES = "feature_states";
+    static final String INDEX_DETAILS = "index_details";
+
+    static final String UNKNOWN_REPO_NAME = "_na_";
 
     private static final Comparator<SnapshotInfo> COMPARATOR = Comparator.comparing(SnapshotInfo::startTime)
         .thenComparing(SnapshotInfo::snapshotId);
 
-    public static final class SnapshotInfoBuilder {
-        private String snapshotName = null;
-        private String snapshotUUID = null;
-        private String repository = UNKNOWN_REPO_NAME;
-        private String state = null;
-        private String reason = null;
-        private List<String> indices = null;
-        private List<String> dataStreams = null;
-        private List<SnapshotFeatureInfo> featureStates = null;
-        private Map<String, IndexSnapshotDetails> indexSnapshotDetails = null;
-        private long startTime = 0L;
-        private long endTime = 0L;
-        private ShardStatsBuilder shardStatsBuilder = null;
-        private Boolean includeGlobalState = null;
-        private Map<String, Object> userMetadata = null;
-        private int version = -1;
-        private List<SnapshotShardFailure> shardFailures = null;
-
-        private void setSnapshotName(String snapshotName) {
-            this.snapshotName = snapshotName;
-        }
-
-        private void setSnapshotUUID(String snapshotUUID) {
-            this.snapshotUUID = snapshotUUID;
-        }
-
-        private void setRepository(String repository) {
-            this.repository = repository;
-        }
-
-        private void setState(String state) {
-            this.state = state;
-        }
-
-        private void setReason(String reason) {
-            this.reason = reason;
-        }
-
-        private void setIndices(List<String> indices) {
-            this.indices = indices;
-        }
-
-        private void setDataStreams(List<String> dataStreams) {
-            this.dataStreams = dataStreams;
-        }
-
-        private void setFeatureStates(List<SnapshotFeatureInfo> featureStates) {
-            this.featureStates = featureStates;
-        }
-
-        private void setIndexSnapshotDetails(Map<String, IndexSnapshotDetails> indexSnapshotDetails) {
-            this.indexSnapshotDetails = indexSnapshotDetails;
-        }
-
-        private void setStartTime(long startTime) {
-            this.startTime = startTime;
-        }
-
-        private void setEndTime(long endTime) {
-            this.endTime = endTime;
-        }
-
-        private void setShardStatsBuilder(ShardStatsBuilder shardStatsBuilder) {
-            this.shardStatsBuilder = shardStatsBuilder;
-        }
-
-        private void setIncludeGlobalState(Boolean includeGlobalState) {
-            this.includeGlobalState = includeGlobalState;
-        }
-
-        private void setUserMetadata(Map<String, Object> userMetadata) {
-            this.userMetadata = userMetadata;
-        }
-
-        private void setVersion(int version) {
-            this.version = version;
-        }
-
-        private void setShardFailures(List<SnapshotShardFailure> shardFailures) {
-            this.shardFailures = shardFailures;
-        }
-
-        public SnapshotInfo build() {
-            final Snapshot snapshot = new Snapshot(repository, new SnapshotId(snapshotName, snapshotUUID));
-
-            if (indices == null) {
-                indices = Collections.emptyList();
-            }
-
-            if (dataStreams == null) {
-                dataStreams = Collections.emptyList();
-            }
-
-            if (featureStates == null) {
-                featureStates = Collections.emptyList();
-            }
-
-            if (indexSnapshotDetails == null) {
-                indexSnapshotDetails = Collections.emptyMap();
-            }
-
-            SnapshotState snapshotState = state == null ? null : SnapshotState.valueOf(state);
-            IndexVersion version = this.version == -1 ? IndexVersion.current() : IndexVersion.fromId(this.version);
-
-            int totalShards = shardStatsBuilder == null ? 0 : shardStatsBuilder.getTotalShards();
-            int successfulShards = shardStatsBuilder == null ? 0 : shardStatsBuilder.getSuccessfulShards();
-
-            if (shardFailures == null) {
-                shardFailures = new ArrayList<>();
-            }
-
-            return new SnapshotInfo(
-                snapshot,
-                indices,
-                dataStreams,
-                featureStates,
-                reason,
-                version,
-                startTime,
-                endTime,
-                totalShards,
-                successfulShards,
-                shardFailures,
-                includeGlobalState,
-                userMetadata,
-                snapshotState,
-                indexSnapshotDetails
-            );
-        }
-    }
-
-    private static final class ShardStatsBuilder {
-        private int totalShards;
-        private int successfulShards;
-
-        private void setTotalShards(int totalShards) {
-            this.totalShards = totalShards;
-        }
-
-        int getTotalShards() {
-            return totalShards;
-        }
-
-        private void setSuccessfulShards(int successfulShards) {
-            this.successfulShards = successfulShards;
-        }
-
-        int getSuccessfulShards() {
-            return successfulShards;
-        }
-    }
-
-    public static final ObjectParser<SnapshotInfoBuilder, Void> SNAPSHOT_INFO_PARSER = new ObjectParser<>(
-        SnapshotInfoBuilder.class.getName(),
-        true,
-        SnapshotInfoBuilder::new
-    );
-
-    private static final ObjectParser<ShardStatsBuilder, Void> SHARD_STATS_PARSER = new ObjectParser<>(
-        ShardStatsBuilder.class.getName(),
-        true,
-        ShardStatsBuilder::new
-    );
-
-    static {
-        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setSnapshotName, new ParseField(SNAPSHOT));
-        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setSnapshotUUID, new ParseField(UUID));
-        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setRepository, new ParseField(REPOSITORY));
-        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setState, new ParseField(STATE));
-        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setReason, new ParseField(REASON));
-        SNAPSHOT_INFO_PARSER.declareStringArray(SnapshotInfoBuilder::setIndices, new ParseField(INDICES));
-        SNAPSHOT_INFO_PARSER.declareStringArray(SnapshotInfoBuilder::setDataStreams, new ParseField(DATA_STREAMS));
-        SNAPSHOT_INFO_PARSER.declareObjectArray(
-            SnapshotInfoBuilder::setFeatureStates,
-            SnapshotFeatureInfo.SNAPSHOT_FEATURE_INFO_PARSER,
-            new ParseField(FEATURE_STATES)
-        );
-        SNAPSHOT_INFO_PARSER.declareObject(
-            SnapshotInfoBuilder::setIndexSnapshotDetails,
-            (p, c) -> p.map(HashMap::new, p2 -> IndexSnapshotDetails.PARSER.parse(p2, c)),
-            new ParseField(INDEX_DETAILS)
-        );
-        SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::setStartTime, new ParseField(START_TIME_IN_MILLIS));
-        SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::setEndTime, new ParseField(END_TIME_IN_MILLIS));
-        SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setShardStatsBuilder, SHARD_STATS_PARSER, new ParseField(SHARDS));
-        SNAPSHOT_INFO_PARSER.declareBoolean(SnapshotInfoBuilder::setIncludeGlobalState, new ParseField(INCLUDE_GLOBAL_STATE));
-        SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setUserMetadata, (p, c) -> p.map(), new ParseField(USER_METADATA));
-        SNAPSHOT_INFO_PARSER.declareInt(SnapshotInfoBuilder::setVersion, new ParseField(VERSION_ID));
-        SNAPSHOT_INFO_PARSER.declareObjectArray(
-            SnapshotInfoBuilder::setShardFailures,
-            SnapshotShardFailure.SNAPSHOT_SHARD_FAILURE_PARSER,
-            new ParseField(FAILURES)
-        );
-
-        SHARD_STATS_PARSER.declareInt(ShardStatsBuilder::setTotalShards, new ParseField(TOTAL));
-        SHARD_STATS_PARSER.declareInt(ShardStatsBuilder::setSuccessfulShards, new ParseField(SUCCESSFUL));
-    }
-
     private final Snapshot snapshot;
 
     @Nullable
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java
index cb3c9a3557d61..87cb67a53fc37 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotInfoTestUtils;
+import org.elasticsearch.snapshots.SnapshotInfoUtils;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
 import org.elasticsearch.test.AbstractXContentTestCase;
 import org.elasticsearch.xcontent.XContentParser;
@@ -30,7 +31,7 @@ public class CreateSnapshotResponseTests extends AbstractXContentTestCase<Create
 
     @Override
     protected CreateSnapshotResponse doParseInstance(XContentParser parser) throws IOException {
-        return CreateSnapshotResponse.fromXContent(parser);
+        return SnapshotInfoUtils.createSnapshotResponseFromXContent(parser);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/SnapshotInfoUtils.java b/test/framework/src/main/java/org/elasticsearch/snapshots/SnapshotInfoUtils.java
new file mode 100644
index 0000000000000..76a31c64ebd4e
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/snapshots/SnapshotInfoUtils.java
@@ -0,0 +1,244 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.snapshots.SnapshotInfo.DATA_STREAMS;
+import static org.elasticsearch.snapshots.SnapshotInfo.END_TIME_IN_MILLIS;
+import static org.elasticsearch.snapshots.SnapshotInfo.FAILURES;
+import static org.elasticsearch.snapshots.SnapshotInfo.FEATURE_STATES;
+import static org.elasticsearch.snapshots.SnapshotInfo.INCLUDE_GLOBAL_STATE;
+import static org.elasticsearch.snapshots.SnapshotInfo.INDEX_DETAILS;
+import static org.elasticsearch.snapshots.SnapshotInfo.INDICES;
+import static org.elasticsearch.snapshots.SnapshotInfo.REASON;
+import static org.elasticsearch.snapshots.SnapshotInfo.REPOSITORY;
+import static org.elasticsearch.snapshots.SnapshotInfo.SHARDS;
+import static org.elasticsearch.snapshots.SnapshotInfo.START_TIME_IN_MILLIS;
+import static org.elasticsearch.snapshots.SnapshotInfo.STATE;
+import static org.elasticsearch.snapshots.SnapshotInfo.SUCCESSFUL;
+import static org.elasticsearch.snapshots.SnapshotInfo.TOTAL;
+import static org.elasticsearch.snapshots.SnapshotInfo.UNKNOWN_REPO_NAME;
+import static org.elasticsearch.snapshots.SnapshotInfo.USER_METADATA;
+import static org.elasticsearch.snapshots.SnapshotInfo.UUID;
+import static org.elasticsearch.snapshots.SnapshotInfo.VERSION_ID;
+import static org.elasticsearch.threadpool.ThreadPool.Names.SNAPSHOT;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class SnapshotInfoUtils {
+
+    private SnapshotInfoUtils() {/* no instances */}
+
+    static final ConstructingObjectParser<CreateSnapshotResponse, Void> CREATE_SNAPSHOT_RESPONSE_PARSER = new ConstructingObjectParser<>(
+        CreateSnapshotResponse.class.getName(),
+        true,
+        args -> new CreateSnapshotResponse(((SnapshotInfoBuilder) args[0]).build())
+    );
+
+    static final ObjectParser<SnapshotInfoBuilder, Void> SNAPSHOT_INFO_PARSER = new ObjectParser<>(
+        SnapshotInfoBuilder.class.getName(),
+        true,
+        SnapshotInfoBuilder::new
+    );
+
+    static final ConstructingObjectParser<ShardStatsBuilder, Void> SHARD_STATS_PARSER = new ConstructingObjectParser<>(
+        ShardStatsBuilder.class.getName(),
+        true,
+        args -> new ShardStatsBuilder((int) Objects.requireNonNullElse(args[0], 0), (int) Objects.requireNonNullElse(args[1], 0))
+    );
+
+    static {
+        SHARD_STATS_PARSER.declareInt(optionalConstructorArg(), new ParseField(TOTAL));
+        SHARD_STATS_PARSER.declareInt(optionalConstructorArg(), new ParseField(SUCCESSFUL));
+
+        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setSnapshotName, new ParseField(SNAPSHOT));
+        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setSnapshotUUID, new ParseField(UUID));
+        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setRepository, new ParseField(REPOSITORY));
+        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setState, new ParseField(STATE));
+        SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setReason, new ParseField(REASON));
+        SNAPSHOT_INFO_PARSER.declareStringArray(SnapshotInfoBuilder::setIndices, new ParseField(INDICES));
+        SNAPSHOT_INFO_PARSER.declareStringArray(SnapshotInfoBuilder::setDataStreams, new ParseField(DATA_STREAMS));
+        SNAPSHOT_INFO_PARSER.declareObjectArray(
+            SnapshotInfoBuilder::setFeatureStates,
+            SnapshotFeatureInfo.SNAPSHOT_FEATURE_INFO_PARSER,
+            new ParseField(FEATURE_STATES)
+        );
+        SNAPSHOT_INFO_PARSER.declareObject(
+            SnapshotInfoBuilder::setIndexSnapshotDetails,
+            (p, c) -> p.map(HashMap::new, p2 -> SnapshotInfo.IndexSnapshotDetails.PARSER.parse(p2, c)),
+            new ParseField(INDEX_DETAILS)
+        );
+        SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::setStartTime, new ParseField(START_TIME_IN_MILLIS));
+        SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::setEndTime, new ParseField(END_TIME_IN_MILLIS));
+        SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setShardStatsBuilder, SHARD_STATS_PARSER, new ParseField(SHARDS));
+        SNAPSHOT_INFO_PARSER.declareBoolean(SnapshotInfoBuilder::setIncludeGlobalState, new ParseField(INCLUDE_GLOBAL_STATE));
+        SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setUserMetadata, (p, c) -> p.map(), new ParseField(USER_METADATA));
+        SNAPSHOT_INFO_PARSER.declareInt(SnapshotInfoBuilder::setVersion, new ParseField(VERSION_ID));
+        SNAPSHOT_INFO_PARSER.declareObjectArray(
+            SnapshotInfoBuilder::setShardFailures,
+            SnapshotShardFailure.SNAPSHOT_SHARD_FAILURE_PARSER,
+            new ParseField(FAILURES)
+        );
+
+        CREATE_SNAPSHOT_RESPONSE_PARSER.declareObject(optionalConstructorArg(), SNAPSHOT_INFO_PARSER, new ParseField("snapshot"));
+    }
+
+    private record ShardStatsBuilder(int totalShards, int successfulShards) {}
+
+    public static final class SnapshotInfoBuilder {
+        private String snapshotName = null;
+        private String snapshotUUID = null;
+        private String repository = UNKNOWN_REPO_NAME;
+        private String state = null;
+        private String reason = null;
+        private List<String> indices = null;
+        private List<String> dataStreams = null;
+        private List<SnapshotFeatureInfo> featureStates = null;
+        private Map<String, SnapshotInfo.IndexSnapshotDetails> indexSnapshotDetails = null;
+        private long startTime = 0L;
+        private long endTime = 0L;
+        private ShardStatsBuilder shardStatsBuilder = null;
+        private Boolean includeGlobalState = null;
+        private Map<String, Object> userMetadata = null;
+        private int version = -1;
+        private List<SnapshotShardFailure> shardFailures = null;
+
+        private void setSnapshotName(String snapshotName) {
+            this.snapshotName = snapshotName;
+        }
+
+        private void setSnapshotUUID(String snapshotUUID) {
+            this.snapshotUUID = snapshotUUID;
+        }
+
+        private void setRepository(String repository) {
+            this.repository = repository;
+        }
+
+        private void setState(String state) {
+            this.state = state;
+        }
+
+        private void setReason(String reason) {
+            this.reason = reason;
+        }
+
+        private void setIndices(List<String> indices) {
+            this.indices = indices;
+        }
+
+        private void setDataStreams(List<String> dataStreams) {
+            this.dataStreams = dataStreams;
+        }
+
+        private void setFeatureStates(List<SnapshotFeatureInfo> featureStates) {
+            this.featureStates = featureStates;
+        }
+
+        private void setIndexSnapshotDetails(Map<String, SnapshotInfo.IndexSnapshotDetails> indexSnapshotDetails) {
+            this.indexSnapshotDetails = indexSnapshotDetails;
+        }
+
+        private void setStartTime(long startTime) {
+            this.startTime = startTime;
+        }
+
+        private void setEndTime(long endTime) {
+            this.endTime = endTime;
+        }
+
+        private void setShardStatsBuilder(ShardStatsBuilder shardStatsBuilder) {
+            this.shardStatsBuilder = shardStatsBuilder;
+        }
+
+        private void setIncludeGlobalState(Boolean includeGlobalState) {
+            this.includeGlobalState = includeGlobalState;
+        }
+
+        private void setUserMetadata(Map<String, Object> userMetadata) {
+            this.userMetadata = userMetadata;
+        }
+
+        private void setVersion(int version) {
+            this.version = version;
+        }
+
+        private void setShardFailures(List<SnapshotShardFailure> shardFailures) {
+            this.shardFailures = shardFailures;
+        }
+
+        public SnapshotInfo build() {
+            final Snapshot snapshot = new Snapshot(repository, new SnapshotId(snapshotName, snapshotUUID));
+
+            if (indices == null) {
+                indices = Collections.emptyList();
+            }
+
+            if (dataStreams == null) {
+                dataStreams = Collections.emptyList();
+            }
+
+            if (featureStates == null) {
+                featureStates = Collections.emptyList();
+            }
+
+            if (indexSnapshotDetails == null) {
+                indexSnapshotDetails = Collections.emptyMap();
+            }
+
+            SnapshotState snapshotState = state == null ? null : SnapshotState.valueOf(state);
+            IndexVersion version = this.version == -1 ? IndexVersion.current() : IndexVersion.fromId(this.version);
+
+            int totalShards = shardStatsBuilder == null ? 0 : shardStatsBuilder.totalShards();
+            int successfulShards = shardStatsBuilder == null ? 0 : shardStatsBuilder.successfulShards();
+
+            if (shardFailures == null) {
+                shardFailures = new ArrayList<>();
+            }
+
+            return new SnapshotInfo(
+                snapshot,
+                indices,
+                dataStreams,
+                featureStates,
+                reason,
+                version,
+                startTime,
+                endTime,
+                totalShards,
+                successfulShards,
+                shardFailures,
+                includeGlobalState,
+                userMetadata,
+                snapshotState,
+                indexSnapshotDetails
+            );
+        }
+    }
+
+    public static CreateSnapshotResponse createSnapshotResponseFromXContent(XContentParser parser) {
+        return CREATE_SNAPSHOT_RESPONSE_PARSER.apply(parser, null);
+    }
+
+    public static SnapshotInfo snapshotInfoFromXContent(XContentParser parser) {
+        return SNAPSHOT_INFO_PARSER.apply(parser, null).build();
+    }
+}
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java
index 729cb8ef47292..f54cd4d4977d7 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
+import org.elasticsearch.snapshots.SnapshotInfoUtils;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
@@ -194,7 +195,9 @@ public void testCreateSnapshotOnTrigger() {
                 assertThat(req.includeGlobalState(), equalTo(globalState));
 
                 try {
-                    return CreateSnapshotResponse.fromXContent(createParser(JsonXContent.jsonXContent, createSnapResponse));
+                    return SnapshotInfoUtils.createSnapshotResponseFromXContent(
+                        createParser(JsonXContent.jsonXContent, createSnapResponse)
+                    );
                 } catch (IOException e) {
                     fail("failed to parse snapshot response");
                     return null;

From faf7b7fd382dbe207a69752436971ca59f32ea5d Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 22 Mar 2024 14:50:01 +0100
Subject: [PATCH 24/31] Add more debugging to 25_id_generation/generates a
 consistent id test (#106667)

Include the _ts_routing_hash field in the search response for more
debugging. In order to add support for docvalue_fields feature for this
field type, a custom DOC_VALUE_FORMAT is needed in order to decode to
valid utf8 string.

Relates #106550
---
 .../test/tsdb/25_id_generation.yml            |  3 +++
 .../TimeSeriesRoutingHashFieldMapper.java     | 25 +++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml
index 04fa2faca209f..621906820e4ad 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml
@@ -110,10 +110,13 @@ generates a consistent id:
           query:
             match_all: {}
           sort: ["@timestamp"]
+          _source: true
+          docvalue_fields: [_ts_routing_hash]
 
   - match: {hits.total.value: 9}
 
   - match: { hits.hits.0._id: cn4excfoxSs_KdA5AAABeRnRFAY }
+  - match: { hits.hits.0.fields._ts_routing_hash: [ cn4exQ ] }
   - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z }
   - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java
index 090fe7839b3e9..d5750600a25c9 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java
@@ -10,6 +10,8 @@
 
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.ByteUtils;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexVersions;
@@ -20,8 +22,10 @@
 import org.elasticsearch.index.fielddata.plain.SortedOrdinalsIndexFieldData;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.field.DelegateDocValuesField;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 
+import java.time.ZoneId;
 import java.util.Base64;
 import java.util.Collections;
 
@@ -43,6 +47,22 @@ public class TimeSeriesRoutingHashFieldMapper extends MetadataFieldMapper {
     static final class TimeSeriesRoutingHashFieldType extends MappedFieldType {
 
         private static final TimeSeriesRoutingHashFieldType INSTANCE = new TimeSeriesRoutingHashFieldType();
+        private static final DocValueFormat DOC_VALUE_FORMAT = new DocValueFormat() {
+
+            @Override
+            public String getWriteableName() {
+                return NAME;
+            }
+
+            @Override
+            public void writeTo(StreamOutput out) {}
+
+            @Override
+            public Object format(BytesRef value) {
+                return Uid.decodeId(value.bytes, value.offset, value.length);
+            }
+
+        };
 
         private TimeSeriesRoutingHashFieldType() {
             super(NAME, false, false, true, TextSearchInfo.NONE, Collections.emptyMap());
@@ -75,6 +95,11 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
         public Query termQuery(Object value, SearchExecutionContext context) {
             throw new IllegalArgumentException("[" + NAME + "] is not searchable");
         }
+
+        @Override
+        public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
+            return DOC_VALUE_FORMAT;
+        }
     }
 
     private TimeSeriesRoutingHashFieldMapper() {

From e92420dc865815995aee526b40dd60c4440a50a7 Mon Sep 17 00:00:00 2001
From: Liam Thompson <32779855+leemthompo@users.noreply.github.com>
Date: Fri, 22 Mar 2024 15:28:30 +0100
Subject: [PATCH 25/31] [DOCS] Update cross cluster search compatability matrix
 (#106677)

---
 .../ccs-version-compat-matrix.asciidoc        | 42 ++++++++++---------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
index bf51042d6adec..5af3c997251dd 100644
--- a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
+++ b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
@@ -1,22 +1,24 @@
-[cols="^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^"]
+[cols="^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^"]
 |====
-| 16+^h| Remote cluster version
+| 17+^h| Remote cluster version
 h| Local cluster version
-            |  6.8        | 7.1–7.16   | 7.17       | 8.0        | 8.1        | 8.2        | 8.3       | 8.4       | 8.5       |8.6         |8.7         |8.8         |8.9         |8.10        |8.11       |8.12 
-| 6.8       |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} 
-| 7.1–7.16  |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} 
-| 7.17      |  {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} 
-| 8.0       |  {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.1       |  {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.2       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.3       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}|{yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.4       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} |{yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.5       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  |{yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.6       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.7       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.8       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.9       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.10      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.11      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}| {yes-icon}
-| 8.12      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon}| {yes-icon}
-|====
\ No newline at end of file
+            |  6.8        | 7.1–7.16   | 7.17       | 8.0        | 8.1        | 8.2        | 8.3       | 8.4       | 8.5       |8.6         |8.7         |8.8         |8.9         |8.10        |8.11       |8.12       |8.13
+| 6.8       |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon}
+| 7.1–7.16  |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon}
+| 7.17      |  {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
+| 8.0       |  {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
+| 8.1       |  {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
+| 8.2       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
+| 8.3       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}|{yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
+| 8.4       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} |{yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
+| 8.5       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon}
+| 8.6       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon}
+| 8.7       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon}
+| 8.8       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon}
+| 8.9       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon}
+| 8.10      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon}
+| 8.11      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon}
+| 8.12      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon}
+| 8.13      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+|====
+

From cc96a8b7e9335682694eea91fb4bd4736a70ecc3 Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Fri, 22 Mar 2024 16:31:51 +0200
Subject: [PATCH 26/31] Add DownsampleMetrics (#106632)

A single instance gets created by the `Downsample` plugin and injected
to related actions.
---
 .../xpack/downsample/Downsample.java          |  6 ++
 .../xpack/downsample/DownsampleMetrics.java   | 74 +++++++++++++++++++
 .../downsample/DownsampleShardIndexer.java    |  9 ++-
 ...DownsampleShardPersistentTaskExecutor.java | 14 +++-
 .../TransportDownsampleIndexerAction.java     |  7 +-
 .../DownsampleActionSingleNodeTests.java      | 25 ++++++-
 6 files changed, 130 insertions(+), 5 deletions(-)
 create mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java

diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java
index 0d3a784e00e53..a6ba4346b1a25 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/Downsample.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.xpack.core.downsample.DownsampleShardPersistentTaskState;
 import org.elasticsearch.xpack.core.downsample.DownsampleShardTask;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
@@ -133,4 +134,9 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             new NamedWriteableRegistry.Entry(PersistentTaskParams.class, DownsampleShardTaskParams.NAME, DownsampleShardTaskParams::new)
         );
     }
+
+    @Override
+    public Collection<?> createComponents(PluginServices services) {
+        return List.of(new DownsampleMetrics(services.telemetryProvider().getMeterRegistry()));
+    }
 }
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java
new file mode 100644
index 0000000000000..576f40a8190f3
--- /dev/null
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleMetrics.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.downsample;
+
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Contains metrics related to downsampling actions.
+ * It gets initialized as a component by the {@link Downsample} plugin, can be injected to its actions.
+ *
+ * In tests, use TestTelemetryPlugin to inject a MeterRegistry for testing purposes
+ * and check that metrics get recorded as expected.
+ *
+ * To add a new metric, you need to:
+ *  - Add a constant for its name, following the naming conventions for metrics.
+ *  - Register it in method {@link #doStart}.
+ *  - Add a function for recording its value.
+ *  - If needed, inject {@link DownsampleMetrics} to the action containing the logic
+ *    that records the metric value. For reference, see {@link TransportDownsampleIndexerAction}.
+ */
+public class DownsampleMetrics extends AbstractLifecycleComponent {
+
+    public static final String LATENCY_SHARD = "es.tsdb.downsample.latency.shard.histogram";
+
+    private final MeterRegistry meterRegistry;
+
+    public DownsampleMetrics(MeterRegistry meterRegistry) {
+        this.meterRegistry = meterRegistry;
+    }
+
+    @Override
+    protected void doStart() {
+        // Register all metrics to track.
+        meterRegistry.registerLongHistogram(LATENCY_SHARD, "Downsampling action latency per shard", "ms");
+    }
+
+    @Override
+    protected void doStop() {}
+
+    @Override
+    protected void doClose() throws IOException {}
+
+    enum ShardActionStatus {
+
+        SUCCESS("success"),
+        MISSING_DOCS("missing_docs"),
+        FAILED("failed");
+
+        public static final String NAME = "status";
+
+        private final String message;
+
+        ShardActionStatus(String message) {
+            this.message = message;
+        }
+
+        String getMessage() {
+            return message;
+        }
+    }
+
+    void recordLatencyShard(long durationInMilliSeconds, ShardActionStatus status) {
+        meterRegistry.getLongHistogram(LATENCY_SHARD).record(durationInMilliSeconds, Map.of(ShardActionStatus.NAME, status.getMessage()));
+    }
+}
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java
index 844c644ee9ea6..773dfbe897b50 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java
@@ -85,6 +85,7 @@ class DownsampleShardIndexer {
     public static final ByteSizeValue DOWNSAMPLE_MAX_BYTES_IN_FLIGHT = new ByteSizeValue(50, ByteSizeUnit.MB);
     private final IndexShard indexShard;
     private final Client client;
+    private final DownsampleMetrics downsampleMetrics;
     private final String downsampleIndex;
     private final Engine.Searcher searcher;
     private final SearchExecutionContext searchExecutionContext;
@@ -103,6 +104,7 @@ class DownsampleShardIndexer {
         final DownsampleShardTask task,
         final Client client,
         final IndexService indexService,
+        final DownsampleMetrics downsampleMetrics,
         final ShardId shardId,
         final String downsampleIndex,
         final DownsampleConfig config,
@@ -113,6 +115,7 @@ class DownsampleShardIndexer {
     ) {
         this.task = task;
         this.client = client;
+        this.downsampleMetrics = downsampleMetrics;
         this.indexShard = indexService.getShard(shardId.id());
         this.downsampleIndex = downsampleIndex;
         this.searcher = indexShard.acquireSearcher("downsampling");
@@ -164,6 +167,7 @@ public DownsampleIndexerAction.ShardDownsampleResponse execute() throws IOExcept
             timeSeriesSearcher.search(initialStateQuery, bucketCollector);
         }
 
+        TimeValue duration = TimeValue.timeValueMillis(client.threadPool().relativeTimeInMillis() - startTime);
         logger.info(
             "Shard [{}] successfully sent [{}], received source doc [{}], indexed downsampled doc [{}], failed [{}], took [{}]",
             indexShard.shardId(),
@@ -171,7 +175,7 @@ public DownsampleIndexerAction.ShardDownsampleResponse execute() throws IOExcept
             task.getNumSent(),
             task.getNumIndexed(),
             task.getNumFailed(),
-            TimeValue.timeValueMillis(client.threadPool().relativeTimeInMillis() - startTime)
+            duration
         );
 
         if (task.getNumIndexed() != task.getNumSent()) {
@@ -187,6 +191,7 @@ public DownsampleIndexerAction.ShardDownsampleResponse execute() throws IOExcept
                 + task.getNumSent()
                 + "]";
             logger.info(error);
+            downsampleMetrics.recordLatencyShard(duration.millis(), DownsampleMetrics.ShardActionStatus.MISSING_DOCS);
             throw new DownsampleShardIndexerException(error, false);
         }
 
@@ -199,6 +204,7 @@ public DownsampleIndexerAction.ShardDownsampleResponse execute() throws IOExcept
                 + task.getNumFailed()
                 + "]";
             logger.info(error);
+            downsampleMetrics.recordLatencyShard(duration.millis(), DownsampleMetrics.ShardActionStatus.FAILED);
             throw new DownsampleShardIndexerException(error, false);
         }
 
@@ -208,6 +214,7 @@ public DownsampleIndexerAction.ShardDownsampleResponse execute() throws IOExcept
             ActionListener.noop()
         );
         logger.info("Downsampling task [" + task.getPersistentTaskId() + " on shard " + indexShard.shardId() + " completed");
+        downsampleMetrics.recordLatencyShard(duration.millis(), DownsampleMetrics.ShardActionStatus.SUCCESS);
         return new DownsampleIndexerAction.ShardDownsampleResponse(indexShard.shardId(), task.getNumIndexed());
     }
 
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java
index b4116d42d25ca..5e6f8b6b5b18e 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java
@@ -188,6 +188,7 @@ private static IndexShardRoutingTable findShardRoutingTable(ShardId shardId, Clu
     static void realNodeOperation(
         Client client,
         IndicesService indicesService,
+        DownsampleMetrics downsampleMetrics,
         DownsampleShardTask task,
         DownsampleShardTaskParams params,
         BytesRef lastDownsampledTsid
@@ -209,6 +210,7 @@ protected void doRun() throws Exception {
                         task,
                         client,
                         indicesService.indexServiceSafe(params.shardId().getIndex()),
+                        downsampleMetrics,
                         params.shardId(),
                         params.downsampleIndex(),
                         params.downsampleConfig(),
@@ -303,17 +305,25 @@ public static class TA extends TransportAction<Request, ActionResponse.Empty> {
 
             private final Client client;
             private final IndicesService indicesService;
+            private final DownsampleMetrics downsampleMetrics;
 
             @Inject
-            public TA(TransportService transportService, ActionFilters actionFilters, Client client, IndicesService indicesService) {
+            public TA(
+                TransportService transportService,
+                ActionFilters actionFilters,
+                Client client,
+                IndicesService indicesService,
+                DownsampleMetrics downsampleMetrics
+            ) {
                 super(NAME, actionFilters, transportService.getTaskManager());
                 this.client = client;
                 this.indicesService = indicesService;
+                this.downsampleMetrics = downsampleMetrics;
             }
 
             @Override
             protected void doExecute(Task t, Request request, ActionListener<ActionResponse.Empty> listener) {
-                realNodeOperation(client, indicesService, request.task, request.params, request.lastDownsampleTsid);
+                realNodeOperation(client, indicesService, downsampleMetrics, request.task, request.params, request.lastDownsampleTsid);
                 listener.onResponse(ActionResponse.Empty.INSTANCE);
             }
         }
diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java
index 24d1df638f80b..f7cfe2d859583 100644
--- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java
+++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java
@@ -52,6 +52,8 @@ public class TransportDownsampleIndexerAction extends TransportBroadcastAction<
     private final ClusterService clusterService;
     private final IndicesService indicesService;
 
+    private final DownsampleMetrics downsampleMetrics;
+
     @Inject
     public TransportDownsampleIndexerAction(
         Client client,
@@ -59,7 +61,8 @@ public TransportDownsampleIndexerAction(
         TransportService transportService,
         IndicesService indicesService,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        DownsampleMetrics downsampleMetrics
     ) {
         super(
             DownsampleIndexerAction.NAME,
@@ -74,6 +77,7 @@ public TransportDownsampleIndexerAction(
         this.client = new OriginSettingClient(client, ClientHelper.ROLLUP_ORIGIN);
         this.clusterService = clusterService;
         this.indicesService = indicesService;
+        this.downsampleMetrics = downsampleMetrics;
     }
 
     @Override
@@ -139,6 +143,7 @@ protected DownsampleIndexerAction.ShardDownsampleResponse shardOperation(
             (DownsampleShardTask) task,
             client,
             indexService,
+            downsampleMetrics,
             request.shardId(),
             request.getDownsampleIndex(),
             request.getRollupConfig(),
diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
index a7b36bbd7dc9b..4c5fdc23e04f9 100644
--- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
+++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
@@ -57,6 +57,7 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchResponseUtils;
@@ -80,6 +81,8 @@
 import org.elasticsearch.tasks.TaskCancelHelper;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -93,6 +96,7 @@
 import org.elasticsearch.xpack.core.ilm.RolloverAction;
 import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.ilm.IndexLifecycle;
+import org.hamcrest.Matchers;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -162,7 +166,8 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
             Downsample.class,
             AggregateMetricMapperPlugin.class,
             DataStreamsPlugin.class,
-            IndexLifecycle.class
+            IndexLifecycle.class,
+            TestTelemetryPlugin.class
         );
     }
 
@@ -623,6 +628,7 @@ public void testCancelDownsampleIndexer() throws IOException {
             task,
             client(),
             indexService,
+            getInstanceFromNode(DownsampleMetrics.class),
             shard.shardId(),
             downsampleIndex,
             config,
@@ -672,6 +678,7 @@ public void testDownsampleBulkFailed() throws IOException {
             task,
             client(),
             indexService,
+            getInstanceFromNode(DownsampleMetrics.class),
             shard.shardId(),
             downsampleIndex,
             config,
@@ -739,6 +746,7 @@ public void testTooManyBytesInFlight() throws IOException {
             task,
             client(),
             indexService,
+            getInstanceFromNode(DownsampleMetrics.class),
             shard.shardId(),
             downsampleIndex,
             config,
@@ -791,6 +799,7 @@ public void testDownsampleStats() throws IOException {
                 task,
                 client(),
                 indexService,
+                getInstanceFromNode(DownsampleMetrics.class),
                 shard.shardId(),
                 downsampleIndex,
                 config,
@@ -810,6 +819,18 @@ public void testDownsampleStats() throws IOException {
 
             assertDownsampleIndexer(indexService, shardNum, task, executeResponse, task.getTotalShardDocCount());
         }
+
+        // Check that metrics get collected as expected.
+        final TestTelemetryPlugin plugin = getInstanceFromNode(PluginsService.class).filterPlugins(TestTelemetryPlugin.class)
+            .findFirst()
+            .orElseThrow();
+        List<Measurement> measurements = plugin.getLongHistogramMeasurement(DownsampleMetrics.LATENCY_SHARD);
+        assertFalse(measurements.isEmpty());
+        for (Measurement measurement : measurements) {
+            assertTrue(measurement.value().toString(), measurement.value().longValue() >= 0 && measurement.value().longValue() < 1000_000);
+            assertEquals(1, measurement.attributes().size());
+            assertThat(measurement.attributes().get("status"), Matchers.in(List.of("success", "failed", "missing_docs")));
+        }
     }
 
     public void testResumeDownsample() throws IOException {
@@ -848,6 +869,7 @@ public void testResumeDownsample() throws IOException {
             task,
             client(),
             indexService,
+            getInstanceFromNode(DownsampleMetrics.class),
             shard.shardId(),
             downsampleIndex,
             config,
@@ -923,6 +945,7 @@ public void testResumeDownsamplePartial() throws IOException {
             task,
             client(),
             indexService,
+            getInstanceFromNode(DownsampleMetrics.class),
             shard.shardId(),
             downsampleIndex,
             config,

From bdce52ebf768658f355168c38fdf9e342389c94b Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 22 Mar 2024 15:33:32 +0100
Subject: [PATCH 27/31] Assert that FsBlobContainer.readBlob does not start
 reading after file length (#106668)

This change adds an assertion in the FsBlobContainer.readBlob to
ensure we are not reading after the last byte of the file.

While this is legal and documented in the SeekableByteChannel.position()
API, having this assertion in place would have caught on CI some regression
introduced recently and only caught on deployments where S3 rejects reads
starting after blob length.
---
 .../org/elasticsearch/common/blobstore/fs/FsBlobContainer.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
index e40ca70460b13..749773cd91eb8 100644
--- a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
+++ b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
@@ -197,6 +197,7 @@ public InputStream readBlob(OperationPurpose purpose, String blobName, long posi
         assert BlobContainer.assertPurposeConsistency(purpose, blobName);
         final SeekableByteChannel channel = Files.newByteChannel(path.resolve(blobName));
         if (position > 0L) {
+            assert position < channel.size() : "reading from " + position + " exceeds file length " + channel.size();
             channel.position(position);
         }
         assert channel.position() == position;

From 0eca8dda92d0b680d9549b2f387677f5f09adf01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20R=C3=BChsen?= <tim.ruehsen@gmx.de>
Date: Fri, 22 Mar 2024 15:53:39 +0100
Subject: [PATCH 28/31] [Profiling] Switch to OTEL cloud.provider,
 cloud.region, host.type (#106656)

* [Profiling] Switch to OTEL cloud.provider, cloud.region, host.type

* Remove wrong spaces from auto-format
---
 .../component-template/profiling-hosts.json   | 25 ++++++--
 .../resources/data/profiling-hosts.ndjson     |  4 +-
 .../xpack/profiling/InstanceType.java         | 64 ++++++++++++-------
 .../ProfilingIndexTemplateRegistry.java       |  5 +-
 .../xpack/profiling/HostMetadataTests.java    | 60 +++++++++++++++--
 .../rest-api-spec/test/profiling/10_basic.yml |  2 +-
 6 files changed, 123 insertions(+), 37 deletions(-)

diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-hosts.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-hosts.json
index f633a8f0cbdb5..353411ed80b2e 100644
--- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-hosts.json
+++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-hosts.json
@@ -33,11 +33,28 @@
           "type": "date",
           "format": "epoch_second"
         },
-        "host.id": {
-          "type": "keyword"
+        "host": {
+          "properties": {
+            "arch": {
+              "type": "keyword"
+            },
+            "id": {
+              "type": "keyword"
+            },
+            "type": {
+              "type": "keyword"
+            }
+          }
         },
-        "host.arch": {
-          "type": "keyword"
+        "cloud": {
+          "properties": {
+            "provider": {
+              "type": "keyword"
+            },
+            "region": {
+              "type": "keyword"
+            }
+          }
         },
         "profiling": {
           "properties": {
diff --git a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-hosts.ndjson b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-hosts.ndjson
index e164f49c4f685..e12a670a79d18 100644
--- a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-hosts.ndjson
+++ b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-hosts.ndjson
@@ -1,4 +1,4 @@
 {"create": {"_index": "profiling-hosts","_id":"eLH27YsBj2lLi3tJYlvr"}}
-{"profiling.project.id":100,"host.id":"8457605156473051743","@timestamp":1700504426,"ecs.version":"1.12.0","profiling.agent.build_timestamp":1688111067,"profiling.instance.private_ipv4s":["192.168.1.2"],"ec2.instance_life_cycle":"on-demand","profiling.agent.config.map_scale_factor":0,"ec2.instance_type":"i3.2xlarge","profiling.host.ip":"192.168.1.2","profiling.agent.config.bpf_log_level":0,"profiling.host.sysctl.net.core.bpf_jit_enable":1,"profiling.agent.config.file":"/etc/prodfiler/prodfiler.conf","ec2.local_ipv4":"192.168.1.2","profiling.agent.config.no_kernel_version_check":false,"host.arch":"amd64","profiling.host.tags":["cloud_provider:aws","cloud_environment:qa","cloud_region:eu-west-1"],"profiling.agent.config.probabilistic_threshold":100,"profiling.agent.config.disable_tls":false,"profiling.agent.config.tracers":"all","profiling.agent.start_time":1700090045589,"profiling.agent.config.max_elements_per_interval":800,"ec2.placement.region":"eu-west-1","profiling.agent.config.present_cpu_cores":8,"profiling.host.kernel_version":"9.9.9-0-aws","profiling.agent.config.bpf_log_size":65536,"profiling.agent.config.known_traces_entries":65536,"profiling.host.sysctl.kernel.unprivileged_bpf_disabled":1,"profiling.agent.config.verbose":false,"profiling.agent.config.probabilistic_interval":"1m0s","ec2.placement.availability_zone_id":"euw1-az1","ec2.security_groups":"","ec2.local_hostname":"ip-192-168-1-2.eu-west-1.compute.internal","ec2.placement.availability_zone":"eu-west-1c","profiling.agent.config.upload_symbols":false,"profiling.host.sysctl.kernel.bpf_stats_enabled":0,"profiling.host.name":"ip-192-168-1-2","ec2.mac":"00:11:22:33:44:55","profiling.host.kernel_proc_version":"Linux version 9.9.9-0-aws","profiling.agent.config.cache_directory":"/var/cache/optimyze/","profiling.agent.version":"v8.12.0","ec2.hostname":"ip-192-168-1-2.eu-west-1.compute.internal","profiling.agent.config.elastic_mode":false,"ec2.ami_id":"ami-aaaaaaaaaaa","ec2.instance_id":"i-0b999999999999999"}
+{"profiling.project.id":100,"host.id":"8457605156473051743","@timestamp":1700504426,"ecs.version":"1.12.0","profiling.agent.build_timestamp":1688111067,"profiling.instance.private_ipv4s":["192.168.1.2"],"ec2.instance_life_cycle":"on-demand","profiling.agent.config.map_scale_factor":0,"host.type":"i3.2xlarge","profiling.host.ip":"192.168.1.2","profiling.agent.config.bpf_log_level":0,"profiling.host.sysctl.net.core.bpf_jit_enable":1,"profiling.agent.config.file":"/etc/prodfiler/prodfiler.conf","ec2.local_ipv4":"192.168.1.2","profiling.agent.config.no_kernel_version_check":false,"host.arch":"amd64","profiling.host.tags":["cloud_provider:aws","cloud_environment:qa","cloud_region:eu-west-1"],"profiling.agent.config.probabilistic_threshold":100,"profiling.agent.config.disable_tls":false,"profiling.agent.config.tracers":"all","profiling.agent.start_time":1700090045589,"profiling.agent.config.max_elements_per_interval":800,"cloud.provider":"aws","cloud.region":"eu-west-1","profiling.agent.config.present_cpu_cores":8,"profiling.host.kernel_version":"9.9.9-0-aws","profiling.agent.config.bpf_log_size":65536,"profiling.agent.config.known_traces_entries":65536,"profiling.host.sysctl.kernel.unprivileged_bpf_disabled":1,"profiling.agent.config.verbose":false,"profiling.agent.config.probabilistic_interval":"1m0s","ec2.placement.availability_zone_id":"euw1-az1","ec2.security_groups":"","ec2.local_hostname":"ip-192-168-1-2.eu-west-1.compute.internal","ec2.placement.availability_zone":"eu-west-1c","profiling.agent.config.upload_symbols":false,"profiling.host.sysctl.kernel.bpf_stats_enabled":0,"profiling.host.name":"ip-192-168-1-2","ec2.mac":"00:11:22:33:44:55","profiling.host.kernel_proc_version":"Linux version 9.9.9-0-aws","profiling.agent.config.cache_directory":"/var/cache/optimyze/","profiling.agent.version":"v8.12.0","ec2.hostname":"ip-192-168-1-2.eu-west-1.compute.internal","profiling.agent.config.elastic_mode":false,"ec2.ami_id":"ami-aaaaaaaaaaa","ec2.instance_id":"i-0b999999999999999"}
 {"create": {"_index": "profiling-hosts", "_id": "u_fHlYwBkmZvQ6tVo1Lr"}}
-{"profiling.project.id":100,"host.id":"7416508186220657211","@timestamp":1703319912,"ecs.version":"1.12.0","profiling.agent.version":"8.11.0","profiling.agent.config.map_scale_factor":0,"profiling.agent.config.probabilistic_threshold":100,"profiling.host.name":"ip-192-186-1-3","profiling.agent.config.no_kernel_version_check":false,"profiling.host.sysctl.net.core.bpf_jit_enable":1,"profiling.agent.config.elastic_mode":false,"azure.compute.vmsize":"Standard_D4s_v3","azure.compute.environment":"AzurePublicCloud","profiling.agent.config.bpf_log_level":0,"profiling.agent.config.known_traces_entries":65536,"profiling.agent.config.ca_address":"example.com:443","profiling.agent.config.tags":"cloud_provider:azure;cloud_environment:qa;cloud_region:eastus2","profiling.host.tags":["cloud_provider:azure","cloud_environment:qa","cloud_region:eastus2"],"profiling.host.kernel_version":"9.9.9-0-azure","profiling.agent.revision":"head-52cc2030","azure.compute.subscriptionid":"1-2-3-4-5","profiling.host.sysctl.kernel.bpf_stats_enabled":0,"host.arch":"amd64","azure.compute.zone":"3","profiling.agent.config.cache_directory":"/var/cache/Elastic/universal-profiling","azure.compute.name":"example-qa-eastus2-001-v1-zone3_6","profiling.agent.config.probabilistic_interval":"1m0s","azure.compute.location":"eastus2","azure.compute.version":"1234.20230510.233254","profiling.instance.private_ipv4s":["192.168.1.3"],"profiling.agent.build_timestamp":1699000836,"profiling.agent.config.file":"/etc/Elastic/universal-profiling/pf-host-agent.conf","profiling.agent.config.bpf_log_size":65536,"profiling.host.sysctl.kernel.unprivileged_bpf_disabled":1,"profiling.agent.config.tracers":"all","profiling.agent.config.present_cpu_cores":4,"profiling.agent.start_time":1702306987358,"profiling.agent.config.disable_tls":false,"azure.compute.ostype":"Linux","profiling.host.ip":"192.168.1.3","profiling.agent.config.max_elements_per_interval":400,"profiling.agent.config.upload_symbols":false,"azure.compute.tags":"bootstrap-version:v1;ece-id:001;environment:qa;identifier:v1;initial-config:;managed-by:terraform;monitored-by:core-infrastructure;owner:core-infrastructure;region_type:ess;role:blueprint;secondary_role:;vars-identifier:eastus2-001-v1","profiling.host.kernel_proc_version":"Linux version 9.9.9-0-azure","profiling.agent.config.verbose":false,"azure.compute.vmid":"1-2-3-4-5"}
+{"profiling.project.id":100,"host.id":"7416508186220657211","@timestamp":1703319912,"ecs.version":"1.12.0","profiling.agent.version":"8.11.0","profiling.agent.config.map_scale_factor":0,"profiling.agent.config.probabilistic_threshold":100,"profiling.host.name":"ip-192-186-1-3","profiling.agent.config.no_kernel_version_check":false,"profiling.host.sysctl.net.core.bpf_jit_enable":1,"profiling.agent.config.elastic_mode":false,"host.type":"Standard_D4s_v3","azure.compute.environment":"AzurePublicCloud","profiling.agent.config.bpf_log_level":0,"profiling.agent.config.known_traces_entries":65536,"profiling.agent.config.ca_address":"example.com:443","profiling.agent.config.tags":"cloud_provider:azure;cloud_environment:qa;cloud_region:eastus2","profiling.host.tags":["cloud_provider:azure","cloud_environment:qa","cloud_region:eastus2"],"profiling.host.kernel_version":"9.9.9-0-azure","profiling.agent.revision":"head-52cc2030","azure.compute.subscriptionid":"1-2-3-4-5","profiling.host.sysctl.kernel.bpf_stats_enabled":0,"host.arch":"amd64","azure.compute.zone":"3","profiling.agent.config.cache_directory":"/var/cache/Elastic/universal-profiling","azure.compute.name":"example-qa-eastus2-001-v1-zone3_6","profiling.agent.config.probabilistic_interval":"1m0s","cloud.provider":"azure","cloud.region":"eastus2","azure.compute.version":"1234.20230510.233254","profiling.instance.private_ipv4s":["192.168.1.3"],"profiling.agent.build_timestamp":1699000836,"profiling.agent.config.file":"/etc/Elastic/universal-profiling/pf-host-agent.conf","profiling.agent.config.bpf_log_size":65536,"profiling.host.sysctl.kernel.unprivileged_bpf_disabled":1,"profiling.agent.config.tracers":"all","profiling.agent.config.present_cpu_cores":4,"profiling.agent.start_time":1702306987358,"profiling.agent.config.disable_tls":false,"azure.compute.ostype":"Linux","profiling.host.ip":"192.168.1.3","profiling.agent.config.max_elements_per_interval":400,"profiling.agent.config.upload_symbols":false,"azure.compute.tags":"bootstrap-version:v1;ece-id:001;environment:qa;identifier:v1;initial-config:;managed-by:terraform;monitored-by:core-infrastructure;owner:core-infrastructure;region_type:ess;role:blueprint;secondary_role:;vars-identifier:eastus2-001-v1","profiling.host.kernel_proc_version":"Linux version 9.9.9-0-azure","profiling.agent.config.verbose":false,"azure.compute.vmid":"1-2-3-4-5"}
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceType.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceType.java
index ee649e381c85d..3aa0a79df13bc 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceType.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/InstanceType.java
@@ -35,6 +35,45 @@ final class InstanceType implements ToXContentObject {
      * @return the {@link InstanceType}
      */
     public static InstanceType fromHostSource(Map<String, Object> source) {
+        String provider = (String) source.get("cloud.provider");
+        if (provider != null) {
+            String region = (String) source.get("cloud.region");
+            String instanceType = (String) source.get("host.type");
+            return new InstanceType(provider, region, instanceType);
+        }
+
+        // Check and handle pre-8.14.0 host sources for backwards-compatibility.
+        InstanceType instanceType = fromObsoleteHostSource(source);
+        if (instanceType != null) {
+            return instanceType;
+        }
+
+        // Support for configured tags (ECS).
+        // Example of tags:
+        // "profiling.host.tags": [
+        // "cloud_provider:aws",
+        // "cloud_environment:qa",
+        // "cloud_region:eu-west-1",
+        // ],
+        String region = null;
+        List<String> tags = listOf(source.get("profiling.host.tags"));
+        for (String tag : tags) {
+            String[] kv = tag.toLowerCase(Locale.ROOT).split(":", 2);
+            if (kv.length != 2) {
+                continue;
+            }
+            if ("cloud_provider".equals(kv[0])) {
+                provider = kv[1];
+            }
+            if ("cloud_region".equals(kv[0])) {
+                region = kv[1];
+            }
+        }
+
+        return new InstanceType(provider, region, null);
+    }
+
+    private static InstanceType fromObsoleteHostSource(Map<String, Object> source) {
         // Check and handle AWS.
         String region = (String) source.get("ec2.placement.region");
         if (region != null) {
@@ -67,30 +106,7 @@ public static InstanceType fromHostSource(Map<String, Object> source) {
             return new InstanceType("azure", region, instanceType);
         }
 
-        // Support for configured tags (ECS).
-        // Example of tags:
-        // "profiling.host.tags": [
-        // "cloud_provider:aws",
-        // "cloud_environment:qa",
-        // "cloud_region:eu-west-1",
-        // ],
-        String provider = null;
-        region = null;
-        List<String> tags = listOf(source.get("profiling.host.tags"));
-        for (String tag : tags) {
-            String[] kv = tag.toLowerCase(Locale.ROOT).split(":", 2);
-            if (kv.length != 2) {
-                continue;
-            }
-            if ("cloud_provider".equals(kv[0])) {
-                provider = kv[1];
-            }
-            if ("cloud_region".equals(kv[0])) {
-                region = kv[1];
-            }
-        }
-
-        return new InstanceType(provider, region, null);
+        return null;
     }
 
     @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistry.java
index 738c06fa310a9..e1698e71afab2 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistry.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/ProfilingIndexTemplateRegistry.java
@@ -47,13 +47,14 @@ public class ProfilingIndexTemplateRegistry extends IndexTemplateRegistry {
     // version 4: Added 'service.name' keyword mapping to profiling-events
     // version 5: Add optional component template '<idx-name>@custom' to all index templates that reference component templates
     // version 6: Added 'host.arch' keyword mapping to profiling-hosts
-    public static final int INDEX_TEMPLATE_VERSION = 6;
+    // version 7: Added 'host.type', 'cloud.provider', 'cloud.region' keyword mappings to profiling-hosts
+    public static final int INDEX_TEMPLATE_VERSION = 7;
 
     // history for individual indices / index templates. Only bump these for breaking changes that require to create a new index
     public static final int PROFILING_EVENTS_VERSION = 2;
     public static final int PROFILING_EXECUTABLES_VERSION = 1;
     public static final int PROFILING_METRICS_VERSION = 1;
-    public static final int PROFILING_HOSTS_VERSION = 1;
+    public static final int PROFILING_HOSTS_VERSION = 2;
     public static final int PROFILING_STACKFRAMES_VERSION = 1;
     public static final int PROFILING_STACKTRACES_VERSION = 1;
     public static final int PROFILING_SYMBOLS_VERSION = 1;
diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/HostMetadataTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/HostMetadataTests.java
index de32754ed69ff..5c24e295909bc 100644
--- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/HostMetadataTests.java
+++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/HostMetadataTests.java
@@ -14,6 +14,32 @@
 
 public class HostMetadataTests extends ESTestCase {
     public void testCreateFromSourceAWS() {
+        final String hostID = "1440256254710195396";
+        final String arch = "amd64";
+        final String provider = "aws";
+        final String region = "eu-west-1";
+        final String instanceType = "md5x.large";
+
+        // tag::noformat
+        HostMetadata host = HostMetadata.fromSource (
+            Map.of (
+                "host.id", hostID,
+                "host.arch", arch,
+                "host.type", instanceType,
+                "cloud.provider", provider,
+                "cloud.region", region
+            )
+        );
+        // end::noformat
+
+        assertEquals(hostID, host.hostID);
+        assertEquals(arch, host.hostArchitecture);
+        assertEquals(provider, host.instanceType.provider);
+        assertEquals(region, host.instanceType.region);
+        assertEquals(instanceType, host.instanceType.name);
+    }
+
+    public void testCreateFromSourceAWSCompat() {
         final String hostID = "1440256254710195396";
         final String arch = "x86_64";
         final String provider = "aws";
@@ -21,8 +47,8 @@ public void testCreateFromSourceAWS() {
         final String instanceType = "md5x.large";
 
         // tag::noformat
-        HostMetadata host = HostMetadata.fromSource(
-            Map.of(
+        HostMetadata host = HostMetadata.fromSource (
+            Map.of (
                 "host.id", hostID,
                 "host.arch", arch,
                 "ec2.instance_type", instanceType,
@@ -39,6 +65,32 @@ public void testCreateFromSourceAWS() {
     }
 
     public void testCreateFromSourceGCP() {
+        final String hostID = "1440256254710195396";
+        final String arch = "amd64";
+        final String provider = "gcp";
+        final String[] regions = { "", "", "europe-west1", "europewest", "europe-west1" };
+
+        for (String region : regions) {
+            // tag::noformat
+            HostMetadata host = HostMetadata.fromSource (
+                Map.of (
+                    "host.id", hostID,
+                    "host.arch", arch,
+                    "cloud.provider", provider,
+                    "cloud.region", region
+                )
+            );
+            // end::noformat
+
+            assertEquals(hostID, host.hostID);
+            assertEquals(arch, host.hostArchitecture);
+            assertEquals(provider, host.instanceType.provider);
+            assertEquals(region, host.instanceType.region);
+            assertEquals("", host.instanceType.name);
+        }
+    }
+
+    public void testCreateFromSourceGCPCompat() {
         final String hostID = "1440256254710195396";
         final String arch = "x86_64";
         final String provider = "gcp";
@@ -142,8 +194,8 @@ public void testCreateFromSourceECS() {
             Map.of(
                 "host.id", hostID,
                 "host.arch", arch,
-                "profiling.host.tags", Arrays.asList(
-                    "cloud_provider:"+provider, "cloud_environment:qa", "cloud_region:"+region)
+                "profiling.host.tags", Arrays.asList (
+                    "cloud_provider:" + provider, "cloud_environment:qa", "cloud_region:" + region)
             )
         );
         // end::noformat
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml
index 4697141bfc599..cc282d26ae418 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml
@@ -116,7 +116,7 @@ setup:
           - {"create": {"_index": "profiling-executables", "_id": "lHp5_WAgpLy2alrUVab6HA"}}
           - {"@timestamp": "1698624000", "Executable": {"build": {"id": "c5f89ea1c68710d2a493bb604c343a92c4f8ddeb"}, "file": {"name": "vmlinux"}}, "Symbolization": {"next_time": "4852491791"}, "ecs": {"version": "1.12.0"}}
           - {"create": {"_index": "profiling-hosts", "_id": "eLH27YsBj2lLi3tJYlvr"}}
-          - {"profiling.project.id": 100, "host.id": "8457605156473051743", "@timestamp": 1700504426, "ecs.version": "1.12.0", "profiling.agent.build_timestamp": 1688111067, "profiling.instance.private_ipv4s": ["192.168.1.2"], "ec2.instance_life_cycle": "on-demand", "profiling.agent.config.map_scale_factor": 0, "ec2.instance_type": "i3.2xlarge", "profiling.host.ip": "192.168.1.2", "profiling.agent.config.bpf_log_level": 0, "profiling.host.sysctl.net.core.bpf_jit_enable": 1, "profiling.agent.config.file": "/etc/prodfiler/prodfiler.conf", "ec2.local_ipv4": "192.168.1.2", "profiling.agent.config.no_kernel_version_check": false, "host.arch": "amd64", "profiling.host.tags": ["cloud_provider:aws", "cloud_environment:qa", "cloud_region:eu-west-1"], "profiling.agent.config.probabilistic_threshold": 100, "profiling.agent.config.disable_tls": false, "profiling.agent.config.tracers": "all", "profiling.agent.start_time": 1700090045589, "profiling.agent.config.max_elements_per_interval": 800, "ec2.placement.region": "eu-west-1", "profiling.agent.config.present_cpu_cores": 8, "profiling.host.kernel_version": "9.9.9-0-aws", "profiling.agent.config.bpf_log_size": 65536, "profiling.agent.config.known_traces_entries": 65536, "profiling.host.sysctl.kernel.unprivileged_bpf_disabled": 1, "profiling.agent.config.verbose": false, "profiling.agent.config.probabilistic_interval": "1m0s", "ec2.placement.availability_zone_id": "euw1-az1", "ec2.security_groups": "", "ec2.local_hostname": "ip-192-168-1-2.eu-west-1.compute.internal", "ec2.placement.availability_zone": "eu-west-1c", "profiling.agent.config.upload_symbols": false, "profiling.host.sysctl.kernel.bpf_stats_enabled": 0, "profiling.host.name": "ip-192-168-1-2", "ec2.mac": "00:11:22:33:44:55", "profiling.host.kernel_proc_version": "Linux version 9.9.9-0-aws", "profiling.agent.config.cache_directory": "/var/cache/optimyze/", "profiling.agent.version": "v8.12.0", "ec2.hostname": "ip-192-168-1-2.eu-west-1.compute.internal", "profiling.agent.config.elastic_mode": false, "ec2.ami_id": "ami-aaaaaaaaaaa", "ec2.instance_id": "i-0b999999999999999" }
+          - {"profiling.project.id": 100, "host.id": "8457605156473051743", "@timestamp": 1700504426, "ecs.version": "1.12.0", "profiling.agent.build_timestamp": 1688111067, "profiling.instance.private_ipv4s": ["192.168.1.2"], "ec2.instance_life_cycle": "on-demand", "profiling.agent.config.map_scale_factor": 0, "host.type": "i3.2xlarge", "profiling.host.ip": "192.168.1.2", "profiling.agent.config.bpf_log_level": 0, "profiling.host.sysctl.net.core.bpf_jit_enable": 1, "profiling.agent.config.file": "/etc/prodfiler/prodfiler.conf", "ec2.local_ipv4": "192.168.1.2", "profiling.agent.config.no_kernel_version_check": false, "host.arch": "amd64", "profiling.host.tags": ["cloud_provider:aws", "cloud_environment:qa", "cloud_region:eu-west-1"], "profiling.agent.config.probabilistic_threshold": 100, "profiling.agent.config.disable_tls": false, "profiling.agent.config.tracers": "all", "profiling.agent.start_time": 1700090045589, "profiling.agent.config.max_elements_per_interval": 800, "cloud.provider": "aws", "cloud.region": "eu-west-1", "profiling.agent.config.present_cpu_cores": 8, "profiling.host.kernel_version": "9.9.9-0-aws", "profiling.agent.config.bpf_log_size": 65536, "profiling.agent.config.known_traces_entries": 65536, "profiling.host.sysctl.kernel.unprivileged_bpf_disabled": 1, "profiling.agent.config.verbose": false, "profiling.agent.config.probabilistic_interval": "1m0s", "ec2.placement.availability_zone_id": "euw1-az1", "ec2.security_groups": "", "ec2.local_hostname": "ip-192-168-1-2.eu-west-1.compute.internal", "ec2.placement.availability_zone": "eu-west-1c", "profiling.agent.config.upload_symbols": false, "profiling.host.sysctl.kernel.bpf_stats_enabled": 0, "profiling.host.name": "ip-192-168-1-2", "ec2.mac": "00:11:22:33:44:55", "profiling.host.kernel_proc_version": "Linux version 9.9.9-0-aws", "profiling.agent.config.cache_directory": "/var/cache/optimyze/", "profiling.agent.version": "v8.12.0", "ec2.hostname": "ip-192-168-1-2.eu-west-1.compute.internal", "profiling.agent.config.elastic_mode": false, "ec2.ami_id": "ami-aaaaaaaaaaa", "ec2.instance_id": "i-0b999999999999999" }
           - {"index": {"_index": "test-events"}}
           - {"@timestamp": "1700504427", "events": ["S07KmaoGhvNte78xwwRbZQ"]}
 ---

From fb17da0647b9dffe00449587dc80171eb14c9a35 Mon Sep 17 00:00:00 2001
From: yashdamani <yashdamani1997@gmail.com>
Date: Fri, 22 Mar 2024 20:36:59 +0530
Subject: [PATCH 29/31] Don't trim highlight snippet if number_of_fragments is
 0

Don't trim highlight snippet if number_of_fragments is 0

Closes #101803
---
 docs/changelog/106306.yaml                    |  6 ++
 .../test/search/510_fragment_trimming_fix.yml | 62 +++++++++++++++++++
 .../uhighlight/CustomPassageFormatter.java    | 12 +++-
 .../highlight/DefaultHighlighter.java         |  7 ++-
 .../CustomPassageFormatterTests.java          |  4 +-
 .../CustomUnifiedHighlighterTests.java        |  2 +-
 6 files changed, 86 insertions(+), 7 deletions(-)
 create mode 100644 docs/changelog/106306.yaml
 create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/510_fragment_trimming_fix.yml

diff --git a/docs/changelog/106306.yaml b/docs/changelog/106306.yaml
new file mode 100644
index 0000000000000..571fe73c31a3e
--- /dev/null
+++ b/docs/changelog/106306.yaml
@@ -0,0 +1,6 @@
+pr: 99961
+summary: "added fix for inconsistent text trimming in Unified Highlighter"
+area: Highlighting
+type: bug
+issues:
+ - 101803
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/510_fragment_trimming_fix.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/510_fragment_trimming_fix.yml
new file mode 100644
index 0000000000000..355ffeebfb1d3
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/510_fragment_trimming_fix.yml
@@ -0,0 +1,62 @@
+setup:
+  - skip:
+        version: ' - 8.13.99'
+        reason: 'no trimming highlight snippets when number_of_fragments is 0 was introduced  in 8.14'
+  - do:
+      indices.create:
+        index: test_trim
+        body:
+          mappings:
+            properties:
+              text:
+                type: text
+                analyzer: whitespace
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: test_trim
+              _id: 1
+          - text: " a  b c d "
+
+---
+"Test unified highlighter with custom passage formatter and number_of_fragments > 0":
+  - do:
+      search:
+        index: test_trim
+        body:
+          query:
+            match:
+              text: "c"
+          highlight:
+            type: unified
+            number_of_fragments: 1
+            fields:
+              text:
+                pre_tags: ["("]
+                post_tags: [")"]
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.highlight.text: ["a  b (c) d"] }
+
+---
+"Test unified highlighter with custom passage formatter when number_of_fragments = 0":
+  - do:
+      search:
+        index: test_trim
+        body:
+          query:
+            match:
+              text: "c"
+          highlight:
+            type: unified
+            number_of_fragments: 0
+            fields:
+              text:
+                pre_tags: ["("]
+                post_tags: [")"]
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.highlight.text: [" a  b (c) d "] }
diff --git a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatter.java b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatter.java
index 6ae2f53a94ad8..41a68494e7cbb 100644
--- a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatter.java
+++ b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatter.java
@@ -23,11 +23,13 @@ public class CustomPassageFormatter extends PassageFormatter {
     private final String preTag;
     private final String postTag;
     private final Encoder encoder;
+    private final int numberOfFragments;
 
-    public CustomPassageFormatter(String preTag, String postTag, Encoder encoder) {
+    public CustomPassageFormatter(String preTag, String postTag, Encoder encoder, int numberOfFragments) {
         this.preTag = preTag;
         this.postTag = postTag;
         this.encoder = encoder;
+        this.numberOfFragments = numberOfFragments;
     }
 
     @Override
@@ -66,8 +68,12 @@ public Snippet[] format(Passage[] passages, String content) {
             } else if (sb.charAt(sb.length() - 1) == HighlightUtils.NULL_SEPARATOR) {
                 sb.deleteCharAt(sb.length() - 1);
             }
-            // and we trim the snippets too
-            snippets[j] = new Snippet(sb.toString().trim(), passage.getScore(), passage.getNumMatches() > 0);
+            // and we trim the snippets too, if the number of fragments > 0
+            if (numberOfFragments == 0) {
+                snippets[j] = new Snippet(sb.toString(), passage.getScore(), passage.getNumMatches() > 0);
+            } else {
+                snippets[j] = new Snippet(sb.toString().trim(), passage.getScore(), passage.getNumMatches() > 0);
+            }
         }
         return snippets;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java
index e77436ba61423..da1be48e6b2c0 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java
@@ -161,7 +161,12 @@ CustomUnifiedHighlighter buildHighlighter(FieldHighlightContext fieldContext) {
     }
 
     protected PassageFormatter getPassageFormatter(SearchHighlightContext.Field field, Encoder encoder) {
-        return new CustomPassageFormatter(field.fieldOptions().preTags()[0], field.fieldOptions().postTags()[0], encoder);
+        return new CustomPassageFormatter(
+            field.fieldOptions().preTags()[0],
+            field.fieldOptions().postTags()[0],
+            encoder,
+            field.fieldOptions().numberOfFragments()
+        );
     }
 
     protected Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) {
diff --git a/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatterTests.java b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatterTests.java
index c1ecaf12828d3..10db924f25f4b 100644
--- a/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatterTests.java
+++ b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomPassageFormatterTests.java
@@ -21,7 +21,7 @@ public class CustomPassageFormatterTests extends ESTestCase {
     public void testSimpleFormat() {
         String content = "This is a really cool highlighter. Unified highlighter gives nice snippets back. No matches here.";
 
-        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<em>", "</em>", new DefaultEncoder());
+        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<em>", "</em>", new DefaultEncoder(), 3);
 
         Passage[] passages = new Passage[3];
         String match = "highlighter";
@@ -62,7 +62,7 @@ public void testSimpleFormat() {
     public void testHtmlEncodeFormat() {
         String content = "<b>This is a really cool highlighter.</b> Unified highlighter gives nice snippets back.";
 
-        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<em>", "</em>", new SimpleHTMLEncoder());
+        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<em>", "</em>", new SimpleHTMLEncoder(), 3);
 
         Passage[] passages = new Passage[2];
         String match = "highlighter";
diff --git a/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
index bf249ba4409ab..8412cc241f51a 100644
--- a/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
+++ b/server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
@@ -145,7 +145,7 @@ private void assertHighlightOneDoc(
                 UnifiedHighlighter.Builder builder = UnifiedHighlighter.builder(searcher, analyzer);
                 builder.withBreakIterator(() -> breakIterator);
                 builder.withFieldMatcher(name -> "text".equals(name));
-                builder.withFormatter(new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder()));
+                builder.withFormatter(new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder(), 3));
                 CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(
                     builder,
                     offsetSource,

From cbc418235abc81c44b8809b971584f53a823629b Mon Sep 17 00:00:00 2001
From: Fernando Briano <fernando@picandocodigo.net>
Date: Fri, 22 Mar 2024 15:25:38 +0000
Subject: [PATCH 30/31] Updates REST API test issue69009.yml (#106663)

---
 .../rest-api-spec/test/search.highlight/issue69009.yml      | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/issue69009.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/issue69009.yml
index 8b2f2f90dd0ee..cd3751dbb9653 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/issue69009.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/issue69009.yml
@@ -29,7 +29,7 @@ setup:
       search:
         index: test
         body: {
-          "query": { "match": { "fox" } },
+          "query": { "match": { "text": "fox" } },
           "highlight": {
             "type": "unified",
             "fields": { "*": { } },
@@ -45,7 +45,7 @@ setup:
       search:
         index: test
         body: {
-          "query": { "match": { "fox" } },
+          "query": { "match": { "text": "fox" } },
           "highlight": {
             "type": "plain",
             "fields": { "*": { } },
@@ -61,7 +61,7 @@ setup:
       search:
         index: test
         body: {
-          "query": { "match": { "fox" } },
+          "query": { "match": { "text": "fox" } },
           "highlight": {
             "type": "fvh",
             "fields": { "*": { } },

From ecb422380fddbb3ea024c64405d8bc529c982d4f Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Fri, 22 Mar 2024 12:48:00 -0400
Subject: [PATCH 31/31] [ESQL] Migrate PropagateEquals optimization (#106627)

Relates to #105217

This copies the PropagateEquals logical optimization into ESQL, following the pattern established in #106499. I've copied the optimization rule into the ESQL version of OptimizerRules, and the tests into OpitmizerRulesTests, and changed the imports &c to point to the appropriate ESQL classes instead of their QL counterparts.

I expect to have several more PRs following this pattern, for the remaining logical optimizations that touch the binary comparison logic. I'm intending to make separate PRs for each, in the interest of making them easier to review.
---
 .../esql/optimizer/LogicalPlanOptimizer.java  |   3 +-
 .../xpack/esql/optimizer/OptimizerRules.java  | 342 ++++++++++++++++
 .../esql/optimizer/OptimizerRulesTests.java   | 383 +++++++++++++++++-
 3 files changed, 715 insertions(+), 13 deletions(-)

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
index 2879173a6f5ad..af8ad7a1fc435 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
@@ -82,7 +82,6 @@
 import static java.util.Collections.singleton;
 import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions;
 import static org.elasticsearch.xpack.ql.expression.Expressions.asAttributes;
-import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateEquals;
 import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.TransformDirection;
 import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.TransformDirection.DOWN;
 
@@ -126,7 +125,7 @@ protected static Batch<LogicalPlan> operators() {
             new BooleanSimplification(),
             new LiteralsOnTheRight(),
             // needs to occur before BinaryComparison combinations (see class)
-            new PropagateEquals(),
+            new org.elasticsearch.xpack.esql.optimizer.OptimizerRules.PropagateEquals(),
             new PropagateNullable(),
             new BooleanFunctionEqualsElimination(),
             new org.elasticsearch.xpack.esql.optimizer.OptimizerRules.CombineDisjunctionsToIn(),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
index e375f11ab3ae7..3ae662580a200 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
@@ -8,7 +8,13 @@
 package org.elasticsearch.xpack.esql.optimizer;
 
 import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.GreaterThan;
+import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.GreaterThanOrEqual;
+import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.LessThan;
+import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NullEquals;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
@@ -34,14 +40,23 @@
 import org.elasticsearch.xpack.ql.expression.AttributeSet;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.Literal;
+import org.elasticsearch.xpack.ql.expression.predicate.Predicates;
+import org.elasticsearch.xpack.ql.expression.predicate.Range;
+import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
+import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.ql.plan.QueryPlan;
 import org.elasticsearch.xpack.ql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.ql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+import org.elasticsearch.xpack.ql.util.CollectionUtils;
 
 import java.time.ZoneId;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.LinkedList;
@@ -50,6 +65,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.xpack.ql.common.Failure.fail;
+import static org.elasticsearch.xpack.ql.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.ql.expression.predicate.Predicates.combineOr;
 import static org.elasticsearch.xpack.ql.expression.predicate.Predicates.splitOr;
 
@@ -240,4 +256,330 @@ protected Expression rule(Or or) {
             return e;
         }
     }
+
+    /**
+     * Propagate Equals to eliminate conjuncted Ranges or BinaryComparisons.
+     * When encountering a different Equals, non-containing {@link Range} or {@link BinaryComparison}, the conjunction becomes false.
+     * When encountering a containing {@link Range}, {@link BinaryComparison} or {@link NotEquals}, these get eliminated by the equality.
+     *
+     * Since this rule can eliminate Ranges and BinaryComparisons, it should be applied before
+     * {@link org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons}.
+     *
+     * This rule doesn't perform any promotion of {@link BinaryComparison}s, that is handled by
+     * {@link org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons} on purpose as the resulting Range might be
+     * foldable (which is picked by the folding rule on the next run).
+     */
+    public static final class PropagateEquals extends org.elasticsearch.xpack.ql.optimizer.OptimizerRules.OptimizerExpressionRule<
+        BinaryLogic> {
+
+        PropagateEquals() {
+            super(org.elasticsearch.xpack.ql.optimizer.OptimizerRules.TransformDirection.DOWN);
+        }
+
+        public Expression rule(BinaryLogic e) {
+            if (e instanceof And) {
+                return propagate((And) e);
+            } else if (e instanceof Or) {
+                return propagate((Or) e);
+            }
+            return e;
+        }
+
+        // combine conjunction
+        private static Expression propagate(And and) {
+            List<Range> ranges = new ArrayList<>();
+            // Only equalities, not-equalities and inequalities with a foldable .right are extracted separately;
+            // the others go into the general 'exps'.
+            // TODO: In 105217, this should change to EsqlBinaryComparison, but it doesn't exist in this branch yet
+            List<BinaryComparison> equals = new ArrayList<>();
+            List<NotEquals> notEquals = new ArrayList<>();
+            List<BinaryComparison> inequalities = new ArrayList<>();
+            List<Expression> exps = new ArrayList<>();
+
+            boolean changed = false;
+
+            for (Expression ex : Predicates.splitAnd(and)) {
+                if (ex instanceof Range) {
+                    ranges.add((Range) ex);
+                } else if (ex instanceof Equals || ex instanceof NullEquals) {
+                    BinaryComparison otherEq = (BinaryComparison) ex;
+                    // equals on different values evaluate to FALSE
+                    // ignore date/time fields as equality comparison might actually be a range check
+                    if (otherEq.right().foldable() && DataTypes.isDateTime(otherEq.left().dataType()) == false) {
+                        for (BinaryComparison eq : equals) {
+                            if (otherEq.left().semanticEquals(eq.left())) {
+                                Integer comp = BinaryComparison.compare(eq.right().fold(), otherEq.right().fold());
+                                if (comp != null) {
+                                    // var cannot be equal to two different values at the same time
+                                    if (comp != 0) {
+                                        return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                    }
+                                }
+                            }
+                        }
+                        equals.add(otherEq);
+                    } else {
+                        exps.add(otherEq);
+                    }
+                } else if (ex instanceof GreaterThan
+                    || ex instanceof GreaterThanOrEqual
+                    || ex instanceof LessThan
+                    || ex instanceof LessThanOrEqual) {
+                        BinaryComparison bc = (BinaryComparison) ex;
+                        if (bc.right().foldable()) {
+                            inequalities.add(bc);
+                        } else {
+                            exps.add(ex);
+                        }
+                    } else if (ex instanceof NotEquals otherNotEq) {
+                        if (otherNotEq.right().foldable()) {
+                            notEquals.add(otherNotEq);
+                        } else {
+                            exps.add(ex);
+                        }
+                    } else {
+                        exps.add(ex);
+                    }
+            }
+
+            // check
+            for (BinaryComparison eq : equals) {
+                Object eqValue = eq.right().fold();
+
+                for (Iterator<Range> iterator = ranges.iterator(); iterator.hasNext();) {
+                    Range range = iterator.next();
+
+                    if (range.value().semanticEquals(eq.left())) {
+                        // if equals is outside the interval, evaluate the whole expression to FALSE
+                        if (range.lower().foldable()) {
+                            Integer compare = BinaryComparison.compare(range.lower().fold(), eqValue);
+                            if (compare != null && (
+                            // eq outside the lower boundary
+                            compare > 0 ||
+                            // eq matches the boundary but should not be included
+                                (compare == 0 && range.includeLower() == false))) {
+                                return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                            }
+                        }
+                        if (range.upper().foldable()) {
+                            Integer compare = BinaryComparison.compare(range.upper().fold(), eqValue);
+                            if (compare != null && (
+                            // eq outside the upper boundary
+                            compare < 0 ||
+                            // eq matches the boundary but should not be included
+                                (compare == 0 && range.includeUpper() == false))) {
+                                return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                            }
+                        }
+
+                        // it's in the range and thus, remove it
+                        iterator.remove();
+                        changed = true;
+                    }
+                }
+
+                // evaluate all NotEquals against the Equal
+                for (Iterator<NotEquals> iter = notEquals.iterator(); iter.hasNext();) {
+                    NotEquals neq = iter.next();
+                    if (eq.left().semanticEquals(neq.left())) {
+                        Integer comp = BinaryComparison.compare(eqValue, neq.right().fold());
+                        if (comp != null) {
+                            if (comp == 0) { // clashing and conflicting: a = 1 AND a != 1
+                                return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                            } else { // clashing and redundant: a = 1 AND a != 2
+                                iter.remove();
+                                changed = true;
+                            }
+                        }
+                    }
+                }
+
+                // evaluate all inequalities against the Equal
+                for (Iterator<BinaryComparison> iter = inequalities.iterator(); iter.hasNext();) {
+                    BinaryComparison bc = iter.next();
+                    if (eq.left().semanticEquals(bc.left())) {
+                        Integer compare = BinaryComparison.compare(eqValue, bc.right().fold());
+                        if (compare != null) {
+                            if (bc instanceof LessThan || bc instanceof LessThanOrEqual) { // a = 2 AND a </<= ?
+                                if ((compare == 0 && bc instanceof LessThan) || // a = 2 AND a < 2
+                                    0 < compare) { // a = 2 AND a </<= 1
+                                    return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                }
+                            } else if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) { // a = 2 AND a >/>= ?
+                                if ((compare == 0 && bc instanceof GreaterThan) || // a = 2 AND a > 2
+                                    compare < 0) { // a = 2 AND a >/>= 3
+                                    return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                }
+                            }
+
+                            iter.remove();
+                            changed = true;
+                        }
+                    }
+                }
+            }
+
+            return changed ? Predicates.combineAnd(CollectionUtils.combine(exps, equals, notEquals, inequalities, ranges)) : and;
+        }
+
+        // combine disjunction:
+        // a = 2 OR a > 3 -> nop; a = 2 OR a > 1 -> a > 1
+        // a = 2 OR a < 3 -> a < 3; a = 2 OR a < 1 -> nop
+        // a = 2 OR 3 < a < 5 -> nop; a = 2 OR 1 < a < 3 -> 1 < a < 3; a = 2 OR 0 < a < 1 -> nop
+        // a = 2 OR a != 2 -> TRUE; a = 2 OR a = 5 -> nop; a = 2 OR a != 5 -> a != 5
+        private static Expression propagate(Or or) {
+            List<Expression> exps = new ArrayList<>();
+            List<Equals> equals = new ArrayList<>(); // foldable right term Equals
+            List<NotEquals> notEquals = new ArrayList<>(); // foldable right term NotEquals
+            List<Range> ranges = new ArrayList<>();
+            List<BinaryComparison> inequalities = new ArrayList<>(); // foldable right term (=limit) BinaryComparision
+
+            // split expressions by type
+            for (Expression ex : Predicates.splitOr(or)) {
+                if (ex instanceof Equals eq) {
+                    if (eq.right().foldable()) {
+                        equals.add(eq);
+                    } else {
+                        exps.add(ex);
+                    }
+                } else if (ex instanceof NotEquals neq) {
+                    if (neq.right().foldable()) {
+                        notEquals.add(neq);
+                    } else {
+                        exps.add(ex);
+                    }
+                } else if (ex instanceof Range) {
+                    ranges.add((Range) ex);
+                } else if (ex instanceof BinaryComparison bc) {
+                    if (bc.right().foldable()) {
+                        inequalities.add(bc);
+                    } else {
+                        exps.add(ex);
+                    }
+                } else {
+                    exps.add(ex);
+                }
+            }
+
+            boolean updated = false; // has the expression been modified?
+
+            // evaluate the impact of each Equal over the different types of Expressions
+            for (Iterator<Equals> iterEq = equals.iterator(); iterEq.hasNext();) {
+                Equals eq = iterEq.next();
+                Object eqValue = eq.right().fold();
+                boolean removeEquals = false;
+
+                // Equals OR NotEquals
+                for (NotEquals neq : notEquals) {
+                    if (eq.left().semanticEquals(neq.left())) { // a = 2 OR a != ? -> ...
+                        Integer comp = BinaryComparison.compare(eqValue, neq.right().fold());
+                        if (comp != null) {
+                            if (comp == 0) { // a = 2 OR a != 2 -> TRUE
+                                return TRUE;
+                            } else { // a = 2 OR a != 5 -> a != 5
+                                removeEquals = true;
+                                break;
+                            }
+                        }
+                    }
+                }
+                if (removeEquals) {
+                    iterEq.remove();
+                    updated = true;
+                    continue;
+                }
+
+                // Equals OR Range
+                for (int i = 0; i < ranges.size(); i++) { // might modify list, so use index loop
+                    Range range = ranges.get(i);
+                    if (eq.left().semanticEquals(range.value())) {
+                        Integer lowerComp = range.lower().foldable() ? BinaryComparison.compare(eqValue, range.lower().fold()) : null;
+                        Integer upperComp = range.upper().foldable() ? BinaryComparison.compare(eqValue, range.upper().fold()) : null;
+
+                        if (lowerComp != null && lowerComp == 0) {
+                            if (range.includeLower() == false) { // a = 2 OR 2 < a < ? -> 2 <= a < ?
+                                ranges.set(
+                                    i,
+                                    new Range(
+                                        range.source(),
+                                        range.value(),
+                                        range.lower(),
+                                        true,
+                                        range.upper(),
+                                        range.includeUpper(),
+                                        range.zoneId()
+                                    )
+                                );
+                            } // else : a = 2 OR 2 <= a < ? -> 2 <= a < ?
+                            removeEquals = true; // update range with lower equality instead or simply superfluous
+                            break;
+                        } else if (upperComp != null && upperComp == 0) {
+                            if (range.includeUpper() == false) { // a = 2 OR ? < a < 2 -> ? < a <= 2
+                                ranges.set(
+                                    i,
+                                    new Range(
+                                        range.source(),
+                                        range.value(),
+                                        range.lower(),
+                                        range.includeLower(),
+                                        range.upper(),
+                                        true,
+                                        range.zoneId()
+                                    )
+                                );
+                            } // else : a = 2 OR ? < a <= 2 -> ? < a <= 2
+                            removeEquals = true; // update range with upper equality instead
+                            break;
+                        } else if (lowerComp != null && upperComp != null) {
+                            if (0 < lowerComp && upperComp < 0) { // a = 2 OR 1 < a < 3
+                                removeEquals = true; // equality is superfluous
+                                break;
+                            }
+                        }
+                    }
+                }
+                if (removeEquals) {
+                    iterEq.remove();
+                    updated = true;
+                    continue;
+                }
+
+                // Equals OR Inequality
+                for (int i = 0; i < inequalities.size(); i++) {
+                    BinaryComparison bc = inequalities.get(i);
+                    if (eq.left().semanticEquals(bc.left())) {
+                        Integer comp = BinaryComparison.compare(eqValue, bc.right().fold());
+                        if (comp != null) {
+                            if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) {
+                                if (comp < 0) { // a = 1 OR a > 2 -> nop
+                                    continue;
+                                } else if (comp == 0 && bc instanceof GreaterThan) { // a = 2 OR a > 2 -> a >= 2
+                                    inequalities.set(i, new GreaterThanOrEqual(bc.source(), bc.left(), bc.right(), bc.zoneId()));
+                                } // else (0 < comp || bc instanceof GreaterThanOrEqual) :
+                                  // a = 3 OR a > 2 -> a > 2; a = 2 OR a => 2 -> a => 2
+
+                                removeEquals = true; // update range with equality instead or simply superfluous
+                                break;
+                            } else if (bc instanceof LessThan || bc instanceof LessThanOrEqual) {
+                                if (comp > 0) { // a = 2 OR a < 1 -> nop
+                                    continue;
+                                }
+                                if (comp == 0 && bc instanceof LessThan) { // a = 2 OR a < 2 -> a <= 2
+                                    inequalities.set(i, new LessThanOrEqual(bc.source(), bc.left(), bc.right(), bc.zoneId()));
+                                } // else (comp < 0 || bc instanceof LessThanOrEqual) : a = 2 OR a < 3 -> a < 3; a = 2 OR a <= 2 -> a <= 2
+                                removeEquals = true; // update range with equality instead or simply superfluous
+                                break;
+                            }
+                        }
+                    }
+                }
+                if (removeEquals) {
+                    iterEq.remove();
+                    updated = true;
+                }
+            }
+
+            return updated ? Predicates.combineOr(CollectionUtils.combine(exps, equals, notEquals, inequalities, ranges)) : or;
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java
index dd9704d57b12a..1aac8efbe6f65 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java
@@ -9,11 +9,19 @@
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.GreaterThan;
+import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.GreaterThanOrEqual;
 import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.LessThan;
+import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NullEquals;
+import org.elasticsearch.xpack.ql.TestUtils;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.Literal;
+import org.elasticsearch.xpack.ql.expression.predicate.Predicates;
+import org.elasticsearch.xpack.ql.expression.predicate.Range;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.ql.plan.logical.Filter;
@@ -24,8 +32,12 @@
 import java.util.List;
 
 import static java.util.Arrays.asList;
-import static org.elasticsearch.xpack.ql.TestUtils.getFieldAttribute;
+import static org.elasticsearch.xpack.ql.TestUtils.equalsOf;
+import static org.elasticsearch.xpack.ql.TestUtils.nullEqualsOf;
+import static org.elasticsearch.xpack.ql.TestUtils.rangeOf;
 import static org.elasticsearch.xpack.ql.TestUtils.relation;
+import static org.elasticsearch.xpack.ql.expression.Literal.FALSE;
+import static org.elasticsearch.xpack.ql.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
 import static org.hamcrest.Matchers.contains;
 
@@ -33,6 +45,8 @@ public class OptimizerRulesTests extends ESTestCase {
     private static final Literal ONE = new Literal(Source.EMPTY, 1, DataTypes.INTEGER);
     private static final Literal TWO = new Literal(Source.EMPTY, 2, DataTypes.INTEGER);
     private static final Literal THREE = new Literal(Source.EMPTY, 3, DataTypes.INTEGER);
+    private static final Literal FOUR = new Literal(Source.EMPTY, 4, DataTypes.INTEGER);
+    private static final Literal FIVE = new Literal(Source.EMPTY, 5, DataTypes.INTEGER);
 
     private static Equals equalsOf(Expression left, Expression right) {
         return new Equals(EMPTY, left, right, null);
@@ -42,11 +56,35 @@ private static LessThan lessThanOf(Expression left, Expression right) {
         return new LessThan(EMPTY, left, right, null);
     }
 
+    public static GreaterThan greaterThanOf(Expression left, Expression right) {
+        return new GreaterThan(EMPTY, left, right, randomZone());
+    }
+
+    public static NotEquals notEqualsOf(Expression left, Expression right) {
+        return new NotEquals(EMPTY, left, right, randomZone());
+    }
+
+    public static NullEquals nullEqualsOf(Expression left, Expression right) {
+        return new NullEquals(EMPTY, left, right, randomZone());
+    }
+
+    public static LessThanOrEqual lessThanOrEqualOf(Expression left, Expression right) {
+        return new LessThanOrEqual(EMPTY, left, right, randomZone());
+    }
+
+    public static GreaterThanOrEqual greaterThanOrEqualOf(Expression left, Expression right) {
+        return new GreaterThanOrEqual(EMPTY, left, right, randomZone());
+    }
+
+    private static FieldAttribute getFieldAttribute() {
+        return TestUtils.getFieldAttribute("a");
+    }
+
     //
     // CombineDisjunction in Equals
     //
     public void testTwoEqualsWithOr() {
-        FieldAttribute fa = getFieldAttribute("a");
+        FieldAttribute fa = getFieldAttribute();
 
         Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO));
         Expression e = new OptimizerRules.CombineDisjunctionsToIn().rule(or);
@@ -57,7 +95,7 @@ public void testTwoEqualsWithOr() {
     }
 
     public void testTwoEqualsWithSameValue() {
-        FieldAttribute fa = getFieldAttribute("a");
+        FieldAttribute fa = getFieldAttribute();
 
         Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, ONE));
         Expression e = new OptimizerRules.CombineDisjunctionsToIn().rule(or);
@@ -68,7 +106,7 @@ public void testTwoEqualsWithSameValue() {
     }
 
     public void testOneEqualsOneIn() {
-        FieldAttribute fa = getFieldAttribute("a");
+        FieldAttribute fa = getFieldAttribute();
 
         Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, List.of(TWO)));
         Expression e = new OptimizerRules.CombineDisjunctionsToIn().rule(or);
@@ -79,7 +117,7 @@ public void testOneEqualsOneIn() {
     }
 
     public void testOneEqualsOneInWithSameValue() {
-        FieldAttribute fa = getFieldAttribute("a");
+        FieldAttribute fa = getFieldAttribute();
 
         Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, asList(ONE, TWO)));
         Expression e = new OptimizerRules.CombineDisjunctionsToIn().rule(or);
@@ -90,7 +128,7 @@ public void testOneEqualsOneInWithSameValue() {
     }
 
     public void testSingleValueInToEquals() {
-        FieldAttribute fa = getFieldAttribute("a");
+        FieldAttribute fa = getFieldAttribute();
 
         Equals equals = equalsOf(fa, ONE);
         Or or = new Or(EMPTY, equals, new In(EMPTY, fa, List.of(ONE)));
@@ -99,7 +137,7 @@ public void testSingleValueInToEquals() {
     }
 
     public void testEqualsBehindAnd() {
-        FieldAttribute fa = getFieldAttribute("a");
+        FieldAttribute fa = getFieldAttribute();
 
         And and = new And(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO));
         Filter dummy = new Filter(EMPTY, relation(), and);
@@ -109,8 +147,8 @@ public void testEqualsBehindAnd() {
     }
 
     public void testTwoEqualsDifferentFields() {
-        FieldAttribute fieldOne = getFieldAttribute("ONE");
-        FieldAttribute fieldTwo = getFieldAttribute("TWO");
+        FieldAttribute fieldOne = TestUtils.getFieldAttribute("ONE");
+        FieldAttribute fieldTwo = TestUtils.getFieldAttribute("TWO");
 
         Or or = new Or(EMPTY, equalsOf(fieldOne, ONE), equalsOf(fieldTwo, TWO));
         Expression e = new OptimizerRules.CombineDisjunctionsToIn().rule(or);
@@ -118,7 +156,7 @@ public void testTwoEqualsDifferentFields() {
     }
 
     public void testMultipleIn() {
-        FieldAttribute fa = getFieldAttribute("a");
+        FieldAttribute fa = getFieldAttribute();
 
         Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), new In(EMPTY, fa, List.of(TWO)));
         Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE)));
@@ -130,7 +168,7 @@ public void testMultipleIn() {
     }
 
     public void testOrWithNonCombinableExpressions() {
-        FieldAttribute fa = getFieldAttribute("a");
+        FieldAttribute fa = getFieldAttribute();
 
         Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), lessThanOf(fa, TWO));
         Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE)));
@@ -143,4 +181,327 @@ public void testOrWithNonCombinableExpressions() {
         assertEquals(fa, in.value());
         assertThat(in.list(), contains(ONE, THREE));
     }
+
+    // a == 1 AND a == 2 -> FALSE
+    public void testDualEqualsConjunction() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq1 = equalsOf(fa, ONE);
+        Equals eq2 = equalsOf(fa, TWO);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq1, eq2));
+        assertEquals(FALSE, exp);
+    }
+
+    // a <=> 1 AND a <=> 2 -> FALSE
+    public void testDualNullEqualsConjunction() {
+        FieldAttribute fa = getFieldAttribute();
+        NullEquals eq1 = nullEqualsOf(fa, ONE);
+        NullEquals eq2 = nullEqualsOf(fa, TWO);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq1, eq2));
+        assertEquals(FALSE, exp);
+    }
+
+    // 1 < a < 10 AND a == 10 -> FALSE
+    public void testEliminateRangeByEqualsOutsideInterval() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq1 = equalsOf(fa, new Literal(EMPTY, 10, DataTypes.INTEGER));
+        Range r = rangeOf(fa, ONE, false, new Literal(EMPTY, 10, DataTypes.INTEGER), false);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq1, r));
+        assertEquals(FALSE, exp);
+    }
+
+    // 1 < a < 10 AND a <=> 10 -> FALSE
+    public void testEliminateRangeByNullEqualsOutsideInterval() {
+        FieldAttribute fa = getFieldAttribute();
+        NullEquals eq1 = nullEqualsOf(fa, new Literal(EMPTY, 10, DataTypes.INTEGER));
+        Range r = rangeOf(fa, ONE, false, new Literal(EMPTY, 10, DataTypes.INTEGER), false);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq1, r));
+        assertEquals(FALSE, exp);
+    }
+
+    // a != 3 AND a = 3 -> FALSE
+    public void testPropagateEquals_VarNeq3AndVarEq3() {
+        FieldAttribute fa = getFieldAttribute();
+        NotEquals neq = notEqualsOf(fa, THREE);
+        Equals eq = equalsOf(fa, THREE);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, neq, eq));
+        assertEquals(FALSE, exp);
+    }
+
+    // a != 4 AND a = 3 -> a = 3
+    public void testPropagateEquals_VarNeq4AndVarEq3() {
+        FieldAttribute fa = getFieldAttribute();
+        NotEquals neq = notEqualsOf(fa, FOUR);
+        Equals eq = equalsOf(fa, THREE);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, neq, eq));
+        assertEquals(Equals.class, exp.getClass());
+        assertEquals(eq, exp);
+    }
+
+    // a = 2 AND a < 2 -> FALSE
+    public void testPropagateEquals_VarEq2AndVarLt2() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        LessThan lt = lessThanOf(fa, TWO);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq, lt));
+        assertEquals(FALSE, exp);
+    }
+
+    // a = 2 AND a <= 2 -> a = 2
+    public void testPropagateEquals_VarEq2AndVarLte2() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        LessThanOrEqual lt = lessThanOrEqualOf(fa, TWO);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq, lt));
+        assertEquals(eq, exp);
+    }
+
+    // a = 2 AND a <= 1 -> FALSE
+    public void testPropagateEquals_VarEq2AndVarLte1() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        LessThanOrEqual lt = lessThanOrEqualOf(fa, ONE);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq, lt));
+        assertEquals(FALSE, exp);
+    }
+
+    // a = 2 AND a > 2 -> FALSE
+    public void testPropagateEquals_VarEq2AndVarGt2() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        GreaterThan gt = greaterThanOf(fa, TWO);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq, gt));
+        assertEquals(FALSE, exp);
+    }
+
+    // a = 2 AND a >= 2 -> a = 2
+    public void testPropagateEquals_VarEq2AndVarGte2() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, TWO);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq, gte));
+        assertEquals(eq, exp);
+    }
+
+    // a = 2 AND a > 3 -> FALSE
+    public void testPropagateEquals_VarEq2AndVarLt3() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        GreaterThan gt = greaterThanOf(fa, THREE);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq, gt));
+        assertEquals(FALSE, exp);
+    }
+
+    // a = 2 AND a < 3 AND a > 1 AND a != 4 -> a = 2
+    public void testPropagateEquals_VarEq2AndVarLt3AndVarGt1AndVarNeq4() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        LessThan lt = lessThanOf(fa, THREE);
+        GreaterThan gt = greaterThanOf(fa, ONE);
+        NotEquals neq = notEqualsOf(fa, FOUR);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression and = Predicates.combineAnd(asList(eq, lt, gt, neq));
+        Expression exp = rule.rule((And) and);
+        assertEquals(eq, exp);
+    }
+
+    // a = 2 AND 1 < a < 3 AND a > 0 AND a != 4 -> a = 2
+    public void testPropagateEquals_VarEq2AndVarRangeGt1Lt3AndVarGt0AndVarNeq4() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        Range range = rangeOf(fa, ONE, false, THREE, false);
+        GreaterThan gt = greaterThanOf(fa, new Literal(EMPTY, 0, DataTypes.INTEGER));
+        NotEquals neq = notEqualsOf(fa, FOUR);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression and = Predicates.combineAnd(asList(eq, range, gt, neq));
+        Expression exp = rule.rule((And) and);
+        assertEquals(eq, exp);
+    }
+
+    // a = 2 OR a > 1 -> a > 1
+    public void testPropagateEquals_VarEq2OrVarGt1() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        GreaterThan gt = greaterThanOf(fa, ONE);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new Or(EMPTY, eq, gt));
+        assertEquals(gt, exp);
+    }
+
+    // a = 2 OR a > 2 -> a >= 2
+    public void testPropagateEquals_VarEq2OrVarGte2() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        GreaterThan gt = greaterThanOf(fa, TWO);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new Or(EMPTY, eq, gt));
+        assertEquals(GreaterThanOrEqual.class, exp.getClass());
+        GreaterThanOrEqual gte = (GreaterThanOrEqual) exp;
+        assertEquals(TWO, gte.right());
+    }
+
+    // a = 2 OR a < 3 -> a < 3
+    public void testPropagateEquals_VarEq2OrVarLt3() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        LessThan lt = lessThanOf(fa, THREE);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new Or(EMPTY, eq, lt));
+        assertEquals(lt, exp);
+    }
+
+    // a = 3 OR a < 3 -> a <= 3
+    public void testPropagateEquals_VarEq3OrVarLt3() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, THREE);
+        LessThan lt = lessThanOf(fa, THREE);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new Or(EMPTY, eq, lt));
+        assertEquals(LessThanOrEqual.class, exp.getClass());
+        LessThanOrEqual lte = (LessThanOrEqual) exp;
+        assertEquals(THREE, lte.right());
+    }
+
+    // a = 2 OR 1 < a < 3 -> 1 < a < 3
+    public void testPropagateEquals_VarEq2OrVarRangeGt1Lt3() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        Range range = rangeOf(fa, ONE, false, THREE, false);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new Or(EMPTY, eq, range));
+        assertEquals(range, exp);
+    }
+
+    // a = 2 OR 2 < a < 3 -> 2 <= a < 3
+    public void testPropagateEquals_VarEq2OrVarRangeGt2Lt3() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        Range range = rangeOf(fa, TWO, false, THREE, false);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new Or(EMPTY, eq, range));
+        assertEquals(Range.class, exp.getClass());
+        Range r = (Range) exp;
+        assertEquals(TWO, r.lower());
+        assertTrue(r.includeLower());
+        assertEquals(THREE, r.upper());
+        assertFalse(r.includeUpper());
+    }
+
+    // a = 3 OR 2 < a < 3 -> 2 < a <= 3
+    public void testPropagateEquals_VarEq3OrVarRangeGt2Lt3() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, THREE);
+        Range range = rangeOf(fa, TWO, false, THREE, false);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new Or(EMPTY, eq, range));
+        assertEquals(Range.class, exp.getClass());
+        Range r = (Range) exp;
+        assertEquals(TWO, r.lower());
+        assertFalse(r.includeLower());
+        assertEquals(THREE, r.upper());
+        assertTrue(r.includeUpper());
+    }
+
+    // a = 2 OR a != 2 -> TRUE
+    public void testPropagateEquals_VarEq2OrVarNeq2() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        NotEquals neq = notEqualsOf(fa, TWO);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new Or(EMPTY, eq, neq));
+        assertEquals(TRUE, exp);
+    }
+
+    // a = 2 OR a != 5 -> a != 5
+    public void testPropagateEquals_VarEq2OrVarNeq5() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq = equalsOf(fa, TWO);
+        NotEquals neq = notEqualsOf(fa, FIVE);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new Or(EMPTY, eq, neq));
+        assertEquals(NotEquals.class, exp.getClass());
+        NotEquals ne = (NotEquals) exp;
+        assertEquals(FIVE, ne.right());
+    }
+
+    // a = 2 OR 3 < a < 4 OR a > 2 OR a!= 2 -> TRUE
+    public void testPropagateEquals_VarEq2OrVarRangeGt3Lt4OrVarGt2OrVarNe2() {
+        FieldAttribute fa = getFieldAttribute();
+        org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals eq = equalsOf(fa, TWO);
+        Range range = rangeOf(fa, THREE, false, FOUR, false);
+        GreaterThan gt = greaterThanOf(fa, TWO);
+        NotEquals neq = notEqualsOf(fa, TWO);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule((Or) Predicates.combineOr(asList(eq, range, neq, gt)));
+        assertEquals(TRUE, exp);
+    }
+
+    // a == 1 AND a == 2 -> nop for date/time fields
+    public void testPropagateEquals_ignoreDateTimeFields() {
+        FieldAttribute fa = TestUtils.getFieldAttribute("a", DataTypes.DATETIME);
+        Equals eq1 = equalsOf(fa, ONE);
+        Equals eq2 = equalsOf(fa, TWO);
+        And and = new And(EMPTY, eq1, eq2);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(and);
+        assertEquals(and, exp);
+    }
+
+    // 1 <= a < 10 AND a == 1 -> a == 1
+    public void testEliminateRangeByEqualsInInterval() {
+        FieldAttribute fa = getFieldAttribute();
+        Equals eq1 = equalsOf(fa, ONE);
+        Range r = rangeOf(fa, ONE, true, new Literal(EMPTY, 10, DataTypes.INTEGER), false);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq1, r));
+        assertEquals(eq1, exp);
+    }
+
+    // 1 <= a < 10 AND a <=> 1 -> a <=> 1
+    public void testEliminateRangeByNullEqualsInInterval() {
+        FieldAttribute fa = getFieldAttribute();
+        NullEquals eq1 = nullEqualsOf(fa, ONE);
+        Range r = rangeOf(fa, ONE, true, new Literal(EMPTY, 10, DataTypes.INTEGER), false);
+
+        OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
+        Expression exp = rule.rule(new And(EMPTY, eq1, r));
+        assertEquals(eq1, exp);
+    }
 }