From c53b0eac8ab9749dfb1a9d4f98097f3ce8c284fa Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Mon, 9 Nov 2020 14:31:52 +0100
Subject: [PATCH 01/34] Fix NPE in toString of FailedShard (#64770)

The concatenation took precedence over the null check, leading to an NPE
because `null` was passed to `ExceptionsHelper.stackTrace(failure))`.
---
 .../elasticsearch/cluster/routing/allocation/FailedShard.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedShard.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedShard.java
index d34e13c507715..7e8ad646c5a73 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedShard.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedShard.java
@@ -43,7 +43,7 @@ public FailedShard(ShardRouting routingEntry, String message, @Nullable Exceptio
     @Override
     public String toString() {
         return "failed shard, shard " + routingEntry + ", message [" + message + "], markAsStale [" + markAsStale + "], failure ["
-            + failure == null ? "null" : ExceptionsHelper.stackTrace(failure) + "]";
+            + (failure == null ? "null" : ExceptionsHelper.stackTrace(failure)) + "]";
     }
 
     /**

From be9725245d13a4ea28ad267872b81b2f7c3c118c Mon Sep 17 00:00:00 2001
From: Henning Andersen <33268011+henningandersen@users.noreply.github.com>
Date: Mon, 9 Nov 2020 15:44:33 +0100
Subject: [PATCH 02/34] Autoscaling delete policy by simple pattern (#64739)

Added the capability to delete autoscaling policies by pattern, allowing
to for instance do:
```
DELETE _autoscaling/policy/*
```
to delete all autoscaling policies. If a wildcard is involved, no
matches are required.
---
 .../apis/delete-autoscaling-policy.asciidoc   | 17 ++++++
 .../autoscaling/delete_autoscaling_policy.yml | 61 +++++++++++++++++++
 ...nsportDeleteAutoscalingPolicyActionIT.java |  8 ++-
 ...TransportPutAutoscalingPolicyActionIT.java | 16 +++++
 .../action/PutAutoscalingPolicyAction.java    | 15 ++++-
 ...ransportDeleteAutoscalingPolicyAction.java | 16 +++--
 ...ortDeleteAutoscalingPolicyActionTests.java | 35 +++++++++++
 7 files changed, 160 insertions(+), 8 deletions(-)

diff --git a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc
index b52d9ece59742..5f06c85587c78 100644
--- a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc
+++ b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc
@@ -61,3 +61,20 @@ The API returns the following result:
   "acknowledged": true
 }
 --------------------------------------------------
+
+This example deletes all autoscaling policies.
+
+[source,console]
+--------------------------------------------------
+DELETE /_autoscaling/policy/*
+--------------------------------------------------
+// TEST
+
+The API returns the following result:
+
+[source,console-result]
+--------------------------------------------------
+{
+  "acknowledged": true
+}
+--------------------------------------------------
diff --git a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml
index 2abf50bf94222..068249ba95c2d 100644
--- a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml
+++ b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml
@@ -31,3 +31,64 @@
       catch: /autoscaling policy with name \[does_not_exist\] does not exist/
       autoscaling.delete_autoscaling_policy:
         name: does_not_exist
+
+---
+"Test delete all non-existing autoscaling policies":
+  - do:
+      autoscaling.delete_autoscaling_policy:
+        name: "*"
+
+---
+"Test delete all existing autoscaling policies":
+  - do:
+      autoscaling.put_autoscaling_policy:
+        name: my_autoscaling_policy_1
+        body:
+          roles: []
+
+  - do:
+      autoscaling.put_autoscaling_policy:
+        name: my_autoscaling_policy_2
+        body:
+          roles: []
+
+  - do:
+      autoscaling.delete_autoscaling_policy:
+        name: "*"
+
+  - do:
+      catch: missing
+      autoscaling.get_autoscaling_policy:
+        name: my_autoscaling_policy_1
+
+  - do:
+      catch: missing
+      autoscaling.get_autoscaling_policy:
+        name: my_autoscaling_policy_2
+
+---
+"Test delete autoscaling policies by wildcard":
+  - do:
+      autoscaling.put_autoscaling_policy:
+        name: my_autoscaling_policy_delete
+        body:
+          roles: []
+
+  - do:
+      autoscaling.put_autoscaling_policy:
+        name: my_autoscaling_policy_keep
+        body:
+          roles: []
+
+  - do:
+      autoscaling.delete_autoscaling_policy:
+        name: "my_autoscaling_policy_delete*"
+
+  - do:
+      catch: missing
+      autoscaling.get_autoscaling_policy:
+        name: my_autoscaling_policy_delete
+
+  - do:
+      autoscaling.get_autoscaling_policy:
+        name: my_autoscaling_policy_keep
diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionIT.java
index 3baadd338684a..94ade2d5fd05f 100644
--- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionIT.java
+++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionIT.java
@@ -30,7 +30,8 @@ public void testDeletePolicy() {
         );
         assertAcked(client().execute(PutAutoscalingPolicyAction.INSTANCE, putRequest).actionGet());
         // we trust that the policy is in the cluster state since we have tests for putting policies
-        final DeleteAutoscalingPolicyAction.Request deleteRequest = new DeleteAutoscalingPolicyAction.Request(policy.name());
+        String deleteName = randomFrom("*", policy.name(), policy.name().substring(0, between(0, policy.name().length())) + "*");
+        final DeleteAutoscalingPolicyAction.Request deleteRequest = new DeleteAutoscalingPolicyAction.Request(deleteName);
         assertAcked(client().execute(DeleteAutoscalingPolicyAction.INSTANCE, deleteRequest).actionGet());
         // now verify that the policy is not in the cluster state
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
@@ -56,4 +57,9 @@ public void testDeleteNonExistentPolicy() {
         assertThat(e.getMessage(), containsString("autoscaling policy with name [" + name + "] does not exist"));
     }
 
+    public void testDeleteNonExistentPolicyByWildcard() {
+        final String name = randomFrom("*", randomAlphaOfLength(8) + "*");
+        final DeleteAutoscalingPolicyAction.Request deleteRequest = new DeleteAutoscalingPolicyAction.Request(name);
+        assertAcked(client().execute(DeleteAutoscalingPolicyAction.INSTANCE, deleteRequest).actionGet());
+    }
 }
diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionIT.java
index 6a785398bbc2a..8b4bfa8760820 100644
--- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionIT.java
+++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionIT.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.xpack.autoscaling.AutoscalingIntegTestCase;
 import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata;
 import org.elasticsearch.xpack.autoscaling.AutoscalingTestCase;
@@ -15,7 +16,10 @@
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.xpack.autoscaling.AutoscalingTestCase.mutateAutoscalingDeciders;
+import static org.elasticsearch.xpack.autoscaling.AutoscalingTestCase.randomAutoscalingDeciders;
 import static org.elasticsearch.xpack.autoscaling.AutoscalingTestCase.randomAutoscalingPolicy;
+import static org.elasticsearch.xpack.autoscaling.AutoscalingTestCase.randomRoles;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.sameInstance;
@@ -57,6 +61,18 @@ public void testNoOpPolicy() {
         );
     }
 
+    public void testPutPolicyIllegalName() {
+        IllegalArgumentException exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> putAutoscalingPolicy(new AutoscalingPolicy(randomAlphaOfLength(8) + "*", randomRoles(), randomAutoscalingDeciders()))
+        );
+
+        assertThat(
+            exception.getMessage(),
+            containsString("name must not contain the following characters " + Strings.INVALID_FILENAME_CHARS)
+        );
+    }
+
     private AutoscalingPolicy putRandomAutoscalingPolicy() {
         final AutoscalingPolicy policy = randomAutoscalingPolicy();
         putAutoscalingPolicy(policy);
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyAction.java
index 2c6e86e2c3b0e..07d016c985ebe 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyAction.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyAction.java
@@ -8,9 +8,11 @@
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.set.Sets;
@@ -136,18 +138,25 @@ public SortedMap<String, AutoscalingDeciderConfiguration> deciders() {
 
         @Override
         public ActionRequestValidationException validate() {
+            ActionRequestValidationException exception = null;
             if (roles != null) {
                 List<String> errors = roles.stream()
                     .filter(Predicate.not(DiscoveryNode.getPossibleRoleNames()::contains))
                     .collect(Collectors.toList());
                 if (errors.isEmpty() == false) {
-                    ActionRequestValidationException exception = new ActionRequestValidationException();
+                    exception = new ActionRequestValidationException();
                     exception.addValidationErrors(errors);
-                    return exception;
                 }
             }
 
-            return null;
+            if (Strings.validFileName(name) == false) {
+                exception = ValidateActions.addValidationError(
+                    "name must not contain the following characters " + Strings.INVALID_FILENAME_CHARS,
+                    exception
+                );
+            }
+
+            return exception;
         }
 
         @Override
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java
index 94e03773a7fc3..e3c772b39870a 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyAction.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -83,13 +84,20 @@ static ClusterState deleteAutoscalingPolicy(final ClusterState currentState, fin
             // we will reject the request below when we try to look up the policy by name
             currentMetadata = AutoscalingMetadata.EMPTY;
         }
-        if (currentMetadata.policies().containsKey(name) == false) {
+        boolean wildcard = Regex.isSimpleMatchPattern(name);
+        if (wildcard == false && currentMetadata.policies().containsKey(name) == false) {
             throw new ResourceNotFoundException("autoscaling policy with name [" + name + "] does not exist");
         }
+
         final SortedMap<String, AutoscalingPolicyMetadata> newPolicies = new TreeMap<>(currentMetadata.policies());
-        final AutoscalingPolicyMetadata policy = newPolicies.remove(name);
-        assert policy != null : name;
-        logger.info("deleting autoscaling policy [{}]", name);
+        if (wildcard) {
+            newPolicies.keySet().removeIf(key -> Regex.simpleMatch(name, key));
+            logger.info("deleting [{}] autoscaling policies", currentMetadata.policies().size() - newPolicies.size());
+        } else {
+            final AutoscalingPolicyMetadata policy = newPolicies.remove(name);
+            assert policy != null : name;
+            logger.info("deleting autoscaling policy [{}]", name);
+        }
         final AutoscalingMetadata newMetadata = new AutoscalingMetadata(newPolicies);
         builder.metadata(Metadata.builder(currentState.getMetadata()).putCustom(AutoscalingMetadata.NAME, newMetadata).build());
         return builder.build();
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java
index 9b4e294875cb8..f0126e321105c 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata;
@@ -103,6 +104,40 @@ public void testDeletePolicy() {
         }
     }
 
+    public void testDeletePolicyByWildcard() {
+        final ClusterState currentState;
+        {
+            final ClusterState.Builder builder = ClusterState.builder(new ClusterName(randomAlphaOfLength(8)));
+            builder.metadata(
+                Metadata.builder().putCustom(AutoscalingMetadata.NAME, randomAutoscalingMetadataOfPolicyCount(randomIntBetween(1, 8)))
+            );
+            currentState = builder.build();
+        }
+        final AutoscalingMetadata currentMetadata = currentState.metadata().custom(AutoscalingMetadata.NAME);
+        final String policyName = randomFrom(currentMetadata.policies().keySet());
+        final String deleteName = randomFrom(policyName.substring(0, between(0, policyName.length()))) + "*";
+        final Logger mockLogger = mock(Logger.class);
+        final ClusterState state = TransportDeleteAutoscalingPolicyAction.deleteAutoscalingPolicy(currentState, deleteName, mockLogger);
+
+        // ensure the policy is deleted from the cluster state
+        final AutoscalingMetadata metadata = state.metadata().custom(AutoscalingMetadata.NAME);
+        assertNotNull(metadata);
+        assertThat(metadata.policies(), not(hasKey(policyName)));
+
+        verify(mockLogger).info("deleting [{}] autoscaling policies", currentMetadata.policies().size() - metadata.policies().size());
+        verifyNoMoreInteractions(mockLogger);
+
+        // ensure that the right policies were preserved
+        for (final Map.Entry<String, AutoscalingPolicyMetadata> entry : currentMetadata.policies().entrySet()) {
+            if (Regex.simpleMatch(deleteName, entry.getKey())) {
+                assertFalse(metadata.policies().containsKey(entry.getKey()));
+            } else {
+                assertThat(metadata.policies(), hasKey(entry.getKey()));
+                assertThat(metadata.policies().get(entry.getKey()).policy(), equalTo(entry.getValue().policy()));
+            }
+        }
+    }
+
     public void testDeleteNonExistentPolicy() {
         final ClusterState currentState;
         {

From 2c342f1ffba5d80510e1727102188adda2c9d57a Mon Sep 17 00:00:00 2001
From: Igor Motov <igor@motovs.org>
Date: Mon, 9 Nov 2020 09:58:45 -0500
Subject: [PATCH 03/34] Add test for bucket sorting to rate agg (#64741)

In the initial implementation I missed the the test to check if bucket
sorting works correctly. This commit adds this test.

Relates to #60674
---
 .../analytics/rate/RateAggregatorTests.java   | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java
index 2878f41da6efe..d6d649b34e4bb 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram;
@@ -354,6 +355,80 @@ public void testKeywordSandwich() throws IOException {
         }, dateType, numType, keywordType);
     }
 
+    public void testKeywordSandwichWithSorting() throws IOException {
+        MappedFieldType numType = new NumberFieldMapper.NumberFieldType("val", NumberFieldMapper.NumberType.INTEGER);
+        MappedFieldType dateType = dateFieldType(DATE_FIELD);
+        MappedFieldType keywordType = new KeywordFieldMapper.KeywordFieldType("term");
+        RateAggregationBuilder rateAggregationBuilder = new RateAggregationBuilder("my_rate").rateUnit("week").field("val");
+        boolean useSum = randomBoolean();
+        if (useSum) {
+            if (randomBoolean()) {
+                rateAggregationBuilder.rateMode("sum");
+            }
+        } else {
+            rateAggregationBuilder.rateMode("value_count");
+        }
+        TermsAggregationBuilder termsAggregationBuilder = new TermsAggregationBuilder("my_term").field("term")
+            .order(BucketOrder.aggregation("my_rate", false))
+            .subAggregation(rateAggregationBuilder);
+        DateHistogramAggregationBuilder dateHistogramAggregationBuilder = new DateHistogramAggregationBuilder("my_date").field(DATE_FIELD)
+            .calendarInterval(new DateHistogramInterval("week"))
+            .subAggregation(termsAggregationBuilder);
+
+        testCase(dateHistogramAggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(
+                doc("2020-11-02T01:07:45", new NumericDocValuesField("val", 1), new SortedSetDocValuesField("term", new BytesRef("a")))
+            );
+            iw.addDocument(
+                doc("2020-11-03T01:07:45", new NumericDocValuesField("val", 2), new SortedSetDocValuesField("term", new BytesRef("a")))
+            );
+            iw.addDocument(
+                doc("2020-11-04T03:43:34", new NumericDocValuesField("val", 4), new SortedSetDocValuesField("term", new BytesRef("b")))
+            );
+            iw.addDocument(
+                doc("2020-11-09T03:43:34", new NumericDocValuesField("val", 30), new SortedSetDocValuesField("term", new BytesRef("a")))
+            );
+            iw.addDocument(
+                doc("2020-11-10T03:43:34", new NumericDocValuesField("val", 4), new SortedSetDocValuesField("term", new BytesRef("b")))
+            );
+            iw.addDocument(
+                doc("2020-11-11T03:43:34", new NumericDocValuesField("val", 4), new SortedSetDocValuesField("term", new BytesRef("b")))
+            );
+        }, (Consumer<InternalDateHistogram>) dh -> {
+            assertThat(dh.getBuckets(), hasSize(2));
+            if (useSum) {
+                StringTerms st1 = (StringTerms) dh.getBuckets().get(0).getAggregations().asList().get(0);
+                assertThat(st1.getBuckets(), hasSize(2));
+                assertThat(st1.getBuckets().get(0).getKeyAsString(), equalTo("b"));
+                assertThat(((InternalRate) st1.getBuckets().get(0).getAggregations().asList().get(0)).value(), closeTo(4.0, 0.000001));
+                assertThat(st1.getBuckets().get(1).getKeyAsString(), equalTo("a"));
+                assertThat(((InternalRate) st1.getBuckets().get(1).getAggregations().asList().get(0)).value(), closeTo(3.0, 0.000001));
+
+                StringTerms st2 = (StringTerms) dh.getBuckets().get(1).getAggregations().asList().get(0);
+                assertThat(st2.getBuckets(), hasSize(2));
+                assertThat(st2.getBuckets().get(0).getKeyAsString(), equalTo("a"));
+                assertThat(((InternalRate) st2.getBuckets().get(0).getAggregations().asList().get(0)).value(), closeTo(30.0, 0.000001));
+                assertThat(st2.getBuckets().get(1).getKeyAsString(), equalTo("b"));
+                assertThat(((InternalRate) st2.getBuckets().get(1).getAggregations().asList().get(0)).value(), closeTo(8.0, 0.000001));
+            } else {
+                StringTerms st1 = (StringTerms) dh.getBuckets().get(0).getAggregations().asList().get(0);
+                assertThat(st1.getBuckets(), hasSize(2));
+                assertThat(st1.getBuckets().get(0).getKeyAsString(), equalTo("a"));
+                assertThat(((InternalRate) st1.getBuckets().get(0).getAggregations().asList().get(0)).value(), closeTo(2.0, 0.000001));
+                assertThat(st1.getBuckets().get(1).getKeyAsString(), equalTo("b"));
+                assertThat(((InternalRate) st1.getBuckets().get(1).getAggregations().asList().get(0)).value(), closeTo(1.0, 0.000001));
+
+                StringTerms st2 = (StringTerms) dh.getBuckets().get(1).getAggregations().asList().get(0);
+                assertThat(st2.getBuckets(), hasSize(2));
+                assertThat(st2.getBuckets().get(0).getKeyAsString(), equalTo("b"));
+                assertThat(((InternalRate) st2.getBuckets().get(0).getAggregations().asList().get(0)).value(), closeTo(2.0, 0.000001));
+                assertThat(st2.getBuckets().get(1).getKeyAsString(), equalTo("a"));
+                assertThat(((InternalRate) st2.getBuckets().get(1).getAggregations().asList().get(0)).value(), closeTo(1.0, 0.000001));
+
+            }
+        }, dateType, numType, keywordType);
+    }
+
     public void testScriptMonthToDay() throws IOException {
         testCase(
             new MatchAllDocsQuery(),

From af3cb43196c9313e02388e618f2e9e0bbaecf4b1 Mon Sep 17 00:00:00 2001
From: Howard <danielhuang@tencent.com>
Date: Mon, 9 Nov 2020 23:04:24 +0800
Subject: [PATCH 04/34] Correct allocation service comment (#60167)

We are shuffling shards, not nodes.
---
 .../cluster/routing/allocation/AllocationService.java     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index a91fe3c5c219e..f6881592a5ad7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -115,7 +115,7 @@ public ClusterState applyStartedShards(ClusterState clusterState, List<ShardRout
             return clusterState;
         }
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
-        // shuffle the unassigned nodes, just so we won't have things like poison failed shards
+        // shuffle the unassigned shards, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
         RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState,
             clusterInfoService.getClusterInfo(), snapshotsInfoService.snapshotShardSizes(), currentNanoTime());
@@ -193,7 +193,7 @@ public ClusterState applyFailedShards(final ClusterState clusterState, final Lis
         ClusterState tmpState = IndexMetadataUpdater.removeStaleIdsWithoutRoutings(clusterState, staleShards, logger);
 
         RoutingNodes routingNodes = getMutableRoutingNodes(tmpState);
-        // shuffle the unassigned nodes, just so we won't have things like poison failed shards
+        // shuffle the unassigned shards, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
         long currentNanoTime = currentNanoTime();
         RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, tmpState,
@@ -248,7 +248,7 @@ public ClusterState applyFailedShards(final ClusterState clusterState, final Lis
      */
     public ClusterState disassociateDeadNodes(ClusterState clusterState, boolean reroute, String reason) {
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
-        // shuffle the unassigned nodes, just so we won't have things like poison failed shards
+        // shuffle the unassigned shards, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
         RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState,
             clusterInfoService.getClusterInfo(), snapshotsInfoService.snapshotShardSizes(), currentNanoTime());
@@ -394,7 +394,7 @@ public ClusterState reroute(ClusterState clusterState, String reason) {
         ClusterState fixedClusterState = adaptAutoExpandReplicas(clusterState);
 
         RoutingNodes routingNodes = getMutableRoutingNodes(fixedClusterState);
-        // shuffle the unassigned nodes, just so we won't have things like poison failed shards
+        // shuffle the unassigned shards, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
         RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, fixedClusterState,
             clusterInfoService.getClusterInfo(), snapshotsInfoService.snapshotShardSizes(), currentNanoTime());

From e712243a2459b48ed07807e4657e0d720c91c500 Mon Sep 17 00:00:00 2001
From: bellengao <gbl_long@163.com>
Date: Mon, 9 Nov 2020 23:19:43 +0800
Subject: [PATCH 05/34] [DOCS] Format the data tier allocation doc (#64722)

---
 .../index-modules/allocation/data_tier_allocation.asciidoc     | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc b/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc
index 7b4e24057c655..ea5aa3c567806 100644
--- a/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc
+++ b/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc
@@ -19,7 +19,7 @@ for data tier filtering.
 
 [discrete]
 [[data-tier-allocation-filters]]
-====Data tier allocation settings
+==== Data tier allocation settings
 
 
 `index.routing.allocation.include._tier`::
@@ -43,7 +43,6 @@ for data tier filtering.
     Assign the index to the first tier in the list that has an available node.
     This prevents indices from remaining unallocated if no nodes are available
     in the preferred tier.
-
     For example, if you set `index.routing.allocation.include._tier_preference`
     to `data_warm,data_hot`, the index is allocated to the warm tier if there
     are nodes with the `data_warm` role. If there are no nodes in the warm tier,

From f9c4d5ff22647d4c3c336ca2aed9d28f94fd0541 Mon Sep 17 00:00:00 2001
From: Henning Andersen <33268011+henningandersen@users.noreply.github.com>
Date: Mon, 9 Nov 2020 16:28:04 +0100
Subject: [PATCH 06/34] Fix autoscaling delete policy rest test (#64796)

Test did not cleanup, now removes the last policy too.
---
 .../test/autoscaling/delete_autoscaling_policy.yml            | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml
index 068249ba95c2d..624bbcb99236f 100644
--- a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml
+++ b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/delete_autoscaling_policy.yml
@@ -92,3 +92,7 @@
   - do:
       autoscaling.get_autoscaling_policy:
         name: my_autoscaling_policy_keep
+
+  - do:
+      autoscaling.delete_autoscaling_policy:
+        name: my_autoscaling_policy_keep

From 678c11c36736cd1e86dbd56c98b586c1ae97e027 Mon Sep 17 00:00:00 2001
From: James Rodewig <40268737+jrodewig@users.noreply.github.com>
Date: Mon, 9 Nov 2020 10:28:25 -0500
Subject: [PATCH 07/34] [DOCS] Remove unneeded spaces (#64759) (#64781)

Co-authored-by: Johannes Mahne <johannes.mahne@elastic.co>
---
 x-pack/docs/en/security/authentication/saml-guide.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/docs/en/security/authentication/saml-guide.asciidoc b/x-pack/docs/en/security/authentication/saml-guide.asciidoc
index 224727824cdd0..f2823032118cf 100644
--- a/x-pack/docs/en/security/authentication/saml-guide.asciidoc
+++ b/x-pack/docs/en/security/authentication/saml-guide.asciidoc
@@ -889,8 +889,8 @@ in {es}. See <<saml-guide-authentication>>
 
 The realm is designed with the assumption that there needs to be a privileged entity
 acting as an authentication proxy. In this case, the custom web application is the
-authentication proxy handling the authentication of end users ( more correctly,
-"delegating" the authentication to the SAML Identity Provider ). The SAML related
+authentication proxy handling the authentication of end users (more correctly,
+"delegating" the authentication to the SAML Identity Provider). The SAML related
 APIs require authentication and the necessary authorization level for the authenticated
 user. For this reason, you must create a Service Account user and assign it a role
 that gives it the `manage_saml` cluster privilege. The use of the `manage_token`

From 357ab0073e61160110592534648561e727674ec7 Mon Sep 17 00:00:00 2001
From: James Rodewig <40268737+jrodewig@users.noreply.github.com>
Date: Mon, 9 Nov 2020 10:36:48 -0500
Subject: [PATCH 08/34] [DOCS] Improve docs for Windows DOS/UNC paths in
 `path.*` settings (#64668)

---
 .../important-settings/path-settings.asciidoc |  58 +++++-----
 .../register-repository.asciidoc              |  71 ++----------
 .../customize-data-log-path-widget.asciidoc   |  39 +++++++
 .../customize-data-log-path.asciidoc          |  22 ++++
 .../tab-widgets/logging-widget.asciidoc       |   2 +-
 docs/reference/tab-widgets/logging.asciidoc   |  18 ++-
 .../multi-data-path-widget.asciidoc           |  39 +++++++
 .../tab-widgets/multi-data-path.asciidoc      |  26 +++++
 .../register-fs-repo-widget.asciidoc          |  39 +++++++
 .../tab-widgets/register-fs-repo.asciidoc     | 103 ++++++++++++++++++
 10 files changed, 323 insertions(+), 94 deletions(-)
 create mode 100644 docs/reference/tab-widgets/customize-data-log-path-widget.asciidoc
 create mode 100644 docs/reference/tab-widgets/customize-data-log-path.asciidoc
 create mode 100644 docs/reference/tab-widgets/multi-data-path-widget.asciidoc
 create mode 100644 docs/reference/tab-widgets/multi-data-path.asciidoc
 create mode 100644 docs/reference/tab-widgets/register-fs-repo-widget.asciidoc
 create mode 100644 docs/reference/tab-widgets/register-fs-repo.asciidoc

diff --git a/docs/reference/setup/important-settings/path-settings.asciidoc b/docs/reference/setup/important-settings/path-settings.asciidoc
index 2867c464e4824..49e6edb356bb0 100644
--- a/docs/reference/setup/important-settings/path-settings.asciidoc
+++ b/docs/reference/setup/important-settings/path-settings.asciidoc
@@ -2,32 +2,32 @@
 [discrete]
 === Path settings
 
-If you are using the `.zip` or `.tar.gz` archives, the `data` and `logs`
-directories are sub-folders of `$ES_HOME`. If these important folders are left
-in their default locations, there is a high risk of them being deleted while
-upgrading {es} to a new version.
-
-In production use, you will almost certainly want to change the locations of the
-`path.data` and `path.logs` folders:
-
-[source,yaml]
---------------------------------------------------
-path:
-  logs: /var/log/elasticsearch
-  data: /var/data/elasticsearch
---------------------------------------------------
-
-The RPM and Debian distributions already use custom paths for `data` and `logs`.
-
-The `path.data` settings can be set to multiple paths, in which case all paths
-will be used to store data. However, the files belonging to a single shard will
-all be stored on the same data path:
-
-[source,yaml]
---------------------------------------------------
-path:
-  data:
-    - /mnt/elasticsearch_1
-    - /mnt/elasticsearch_2
-    - /mnt/elasticsearch_3
---------------------------------------------------
+For <<targz,macOS `.tar.gz`>>, <<targz,Linux `.tar.gz`>>, and
+<<zip-windows,Windows `.zip`>> installations, {es} writes data and logs to the
+respective `data` and `logs` subdirectories of `$ES_HOME` by default.
+However, files in `$ES_HOME` risk deletion during an upgrade.
+
+In production, we strongly recommend you set the `path.data` and `path.logs` in
+`elasticsearch.yml` to locations outside of `$ES_HOME`.
+
+TIP: <<docker,Docker>>, <<deb,Debian>>, <<rpm,RPM>>, <<brew,macOS Homebrew>>,
+and <<windows,Windows `.msi`>> installations write data and log to locations
+outside of `$ES_HOME` by default.
+
+Supported `path.data` and `path.logs` values vary by platform:
+
+include::{es-repo-dir}/tab-widgets/code.asciidoc[]
+
+include::{es-repo-dir}/tab-widgets/customize-data-log-path-widget.asciidoc[]
+
+If needed, you can specify multiple paths in `path.data`. {es} stores the node's
+data across all provided paths but keeps each shard's data on the same path.
+
+WARNING: {es} does not balance shards across a node's data paths. High disk
+usage in a single path can trigger a <<disk-based-shard-allocation,high disk
+usage watermark>> for the entire node. If triggered, {es} will not add shards to
+the node, even if the node’s other paths have available disk space. If you need
+additional disk space, we recommend you add a new node rather than additional
+data paths. 
+
+include::{es-repo-dir}/tab-widgets/multi-data-path-widget.asciidoc[]
\ No newline at end of file
diff --git a/docs/reference/snapshot-restore/register-repository.asciidoc b/docs/reference/snapshot-restore/register-repository.asciidoc
index f0ae042cb349c..42a0f10c8bbd2 100644
--- a/docs/reference/snapshot-restore/register-repository.asciidoc
+++ b/docs/reference/snapshot-restore/register-repository.asciidoc
@@ -100,71 +100,20 @@ are left untouched and in place.
 [[snapshots-filesystem-repository]]
 === Shared file system repository
 
-The shared file system repository (`"type": "fs"`) uses the shared file system to store snapshots. In order to register
-the shared file system repository it is necessary to mount the same shared filesystem to the same location on all
-master and data nodes. This location (or one of its parent directories) must be registered in the `path.repo`
-setting on all master and data nodes.
+Use a shared file system repository (`"type": "fs"`) to store snapshots on a
+shared file system.
 
-Assuming that the shared filesystem is mounted to `/mount/backups/my_fs_backup_location`, the following setting should
-be added to `elasticsearch.yml` file:
+To register a shared file system repository, first mount the file system to the
+same location on all master and data nodes. Then add the file system's
+path or parent directory to the `path.repo` setting in `elasticsearch.yml` for
+each master and data node. For running clusters, this requires a
+<<restart-cluster-rolling,rolling restart>> of each node.
 
-[source,yaml]
---------------
-path.repo: ["/mount/backups", "/mount/longterm_backups"]
---------------
-
-The `path.repo` setting supports Microsoft Windows UNC paths as long as at least server name and share are specified as
-a prefix and back slashes are properly escaped:
-
-[source,yaml]
---------------
-path.repo: ["\\\\MY_SERVER\\Snapshots"]
---------------
-
-After all nodes are restarted, the following command can be used to register the shared file system repository with
-the name `my_fs_backup`:
-
-[source,console]
------------------------------------
-PUT /_snapshot/my_fs_backup
-{
-  "type": "fs",
-  "settings": {
-    "location": "/mount/backups/my_fs_backup_location",
-    "compress": true
-  }
-}
------------------------------------
-// TEST[skip:no access to absolute path]
-
-If the repository location is specified as a relative path this path will be resolved against the first path specified
-in `path.repo`:
+Supported `path.repo` values vary by platform:
 
-[source,console]
------------------------------------
-PUT /_snapshot/my_fs_backup
-{
-  "type": "fs",
-  "settings": {
-    "location": "my_fs_backup_location",
-    "compress": true
-  }
-}
------------------------------------
-// TEST[continued]
+include::{es-repo-dir}/tab-widgets/code.asciidoc[]
 
-The following settings are supported:
-
-`location`:: Location of the snapshots. Mandatory.
-`compress`:: Turns on compression of the snapshot files. Compression is applied only to metadata files (index mapping and settings). Data files are not compressed. Defaults to `true`.
-`chunk_size`:: Big files can be broken down into chunks during snapshotting if needed. Specify the chunk size as a value and
-unit, for example: `1GB`, `10MB`, `5KB`, `500B`. Defaults to `null` (unlimited chunk size).
-`max_restore_bytes_per_sec`:: Throttles per node restore rate. Defaults to unlimited. Note that restores are also throttled through <<recovery,recovery settings>>.
-`max_snapshot_bytes_per_sec`:: Throttles per node snapshot rate. Defaults to `40mb` per second.
-`readonly`:: Makes repository read-only.  Defaults to `false`.
-`max_number_of_snapshots`:: Limits the maximum number of snapshots that the repository may contain. Defaults to `500`. Note that snapshot repositories do not
-scale indefinitely in size and might lead to master node performance and stability issues if they grow past a certain size. We do not recommend increasing this setting.
-Instead you should delete older snapshots or use multiple repositories.
+include::{es-repo-dir}/tab-widgets/register-fs-repo-widget.asciidoc[]
 
 [discrete]
 [[snapshots-read-only-repository]]
diff --git a/docs/reference/tab-widgets/customize-data-log-path-widget.asciidoc b/docs/reference/tab-widgets/customize-data-log-path-widget.asciidoc
new file mode 100644
index 0000000000000..a87f4c6d5576d
--- /dev/null
+++ b/docs/reference/tab-widgets/customize-data-log-path-widget.asciidoc
@@ -0,0 +1,39 @@
+++++
+<div class="tabs" data-tab-group="os">
+  <div role="tablist" aria-label="data-log-path">
+    <button role="tab"
+            aria-selected="true"
+            aria-controls="unix-tab-data-log-path"
+            id="unix-data-log-path">
+      Unix-like systems
+    </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="win-tab-data-log-path"
+            id="win-data-log-path">
+      Windows
+    </button>
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="unix-tab-data-log-path"
+       aria-labelledby="unix-data-log-path">
+++++
+
+include::customize-data-log-path.asciidoc[tag=unix]
+
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="win-tab-data-log-path"
+       aria-labelledby="win-data-log-path"
+       hidden="">
+++++
+
+include::customize-data-log-path.asciidoc[tag=win]
+
+++++
+  </div>
+</div>
+++++
\ No newline at end of file
diff --git a/docs/reference/tab-widgets/customize-data-log-path.asciidoc b/docs/reference/tab-widgets/customize-data-log-path.asciidoc
new file mode 100644
index 0000000000000..45cee2babeb29
--- /dev/null
+++ b/docs/reference/tab-widgets/customize-data-log-path.asciidoc
@@ -0,0 +1,22 @@
+// tag::unix[]
+Linux and macOS installations support Unix-style paths:
+
+[source,yaml]
+----
+path:
+  data: /var/data/elasticsearch
+  logs: /var/log/elasticsearch
+----
+// end::unix[]
+
+
+// tag::win[]
+Windows installations support DOS paths with escaped backslashes:
+
+[source,yaml]
+----
+path:
+  data: "C:\\Elastic\\Elasticsearch\\data"
+  logs: "C:\\Elastic\\Elasticsearch\\logs"
+----
+// end::win[]
\ No newline at end of file
diff --git a/docs/reference/tab-widgets/logging-widget.asciidoc b/docs/reference/tab-widgets/logging-widget.asciidoc
index 8b20c525ca1e9..b88633bdb48a8 100644
--- a/docs/reference/tab-widgets/logging-widget.asciidoc
+++ b/docs/reference/tab-widgets/logging-widget.asciidoc
@@ -25,7 +25,7 @@
             aria-controls="mac-tab-logs"
             id="mac-logs"
             tabindex="-1">
-      MacOS
+      macOS
     </button>
     <button role="tab"
             aria-selected="false"
diff --git a/docs/reference/tab-widgets/logging.asciidoc b/docs/reference/tab-widgets/logging.asciidoc
index f400db3963adc..7570a1c92551e 100644
--- a/docs/reference/tab-widgets/logging.asciidoc
+++ b/docs/reference/tab-widgets/logging.asciidoc
@@ -12,26 +12,38 @@ For <<rpm,RPM installations>>, {es} writes logs to `/var/log/elasticsearch`.
 // end::rpm[]
 
 // tag::mac[]
-For <<targz,MacOS `.tar.gz`>> installations, {es} writes logs to
+For <<targz,macOS `.tar.gz`>> installations, {es} writes logs to
 `$ES_HOME/logs`.
+
+Files in `$ES_HOME` risk deletion during an upgrade. In production, we strongly
+recommend you set `path.logs` to a location outside of `$ES_HOME`.
+See <<path-settings>>.
 // end::mac[]
 
 // tag::brew[]
-For <<brew,MacOS Homebrew>> installations, {es} writes logs to
+For <<brew,macOS Homebrew>> installations, {es} writes logs to
 `/usr/local/var/log/elasticsearch`.
 // end::brew[]
 
 // tag::linux[]
 For <<targz,Linux `.tar.gz`>> installations, {es} writes logs to
 `$ES_HOME/logs`.
+
+Files in `$ES_HOME` risk deletion during an upgrade. In production, we strongly
+recommend you set `path.logs` to a location outside of `$ES_HOME`.
+See <<path-settings>>.
 // end::linux[]
 
 // tag::win-zip[]
 For <<zip-windows,Windows `.zip`>> installations, {es} writes logs to
 `%ES_HOME%\logs`.
+
+Files in `%ES_HOME%` risk deletion during an upgrade. In production, we strongly
+recommend you set `path.logs` to a location outside of `%ES_HOME%``.
+See <<path-settings>>.
 // end::win-zip[]
 
 // tag::win-msi[]
 For <<windows,Windows `.msi`>> installations, {es} writes logs to
 `%ALLUSERSPROFILE%\Elastic\Elasticsearch\logs`.
-// end::win-msi[]
\ No newline at end of file
+// end::win-msi[]
diff --git a/docs/reference/tab-widgets/multi-data-path-widget.asciidoc b/docs/reference/tab-widgets/multi-data-path-widget.asciidoc
new file mode 100644
index 0000000000000..f665e7a1cc912
--- /dev/null
+++ b/docs/reference/tab-widgets/multi-data-path-widget.asciidoc
@@ -0,0 +1,39 @@
+++++
+<div class="tabs" data-tab-group="os">
+  <div role="tablist" aria-label="multi-data-path">
+    <button role="tab"
+            aria-selected="true"
+            aria-controls="unix-tab-multi-data-path"
+            id="unix-multi-data-path">
+      Unix-like systems
+    </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="win-tab-multi-data-path"
+            id="win-multi-data-path">
+      Windows
+    </button>
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="unix-tab-multi-data-path"
+       aria-labelledby="unix-multi-data-path">
+++++
+
+include::multi-data-path.asciidoc[tag=unix]
+
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="win-tab-multi-data-path"
+       aria-labelledby="win-multi-data-path"
+       hidden="">
+++++
+
+include::multi-data-path.asciidoc[tag=win]
+
+++++
+  </div>
+</div>
+++++
\ No newline at end of file
diff --git a/docs/reference/tab-widgets/multi-data-path.asciidoc b/docs/reference/tab-widgets/multi-data-path.asciidoc
new file mode 100644
index 0000000000000..0a63c7791f66c
--- /dev/null
+++ b/docs/reference/tab-widgets/multi-data-path.asciidoc
@@ -0,0 +1,26 @@
+// tag::unix[]
+Linux and macOS installations support multiple Unix-style paths in `path.data`:
+
+[source,yaml]
+----
+path:
+  data:
+    - /mnt/elasticsearch_1
+    - /mnt/elasticsearch_2
+    - /mnt/elasticsearch_3
+----
+// end::unix[]
+
+
+// tag::win[]
+Windows installations support multiple DOS paths in `path.data`:
+
+[source,yaml]
+----
+path:
+  data:
+    - "C:\\Elastic\\Elasticsearch_1"
+    - "E:\\Elastic\\Elasticsearch_1"
+    - "F:\\Elastic\\Elasticsearch_3"
+----
+// end::win[]
diff --git a/docs/reference/tab-widgets/register-fs-repo-widget.asciidoc b/docs/reference/tab-widgets/register-fs-repo-widget.asciidoc
new file mode 100644
index 0000000000000..d0ac7041adf2b
--- /dev/null
+++ b/docs/reference/tab-widgets/register-fs-repo-widget.asciidoc
@@ -0,0 +1,39 @@
+++++
+<div class="tabs" data-tab-group="os">
+  <div role="tablist" aria-label="fs-repo">
+    <button role="tab"
+            aria-selected="true"
+            aria-controls="unix-tab-fs-repo"
+            id="unix-fs-repo">
+      Unix-like systems
+    </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="win-tab-fs-repo"
+            id="win-fs-repo">
+      Windows
+    </button>
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="unix-tab-fs-repo"
+       aria-labelledby="unix-fs-repo">
+++++
+
+include::register-fs-repo.asciidoc[tag=unix]
+
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="win-tab-fs-repo"
+       aria-labelledby="win-fs-repo"
+       hidden="">
+++++
+
+include::register-fs-repo.asciidoc[tag=win]
+
+++++
+  </div>
+</div>
+++++
\ No newline at end of file
diff --git a/docs/reference/tab-widgets/register-fs-repo.asciidoc b/docs/reference/tab-widgets/register-fs-repo.asciidoc
new file mode 100644
index 0000000000000..bbfeabcbf571c
--- /dev/null
+++ b/docs/reference/tab-widgets/register-fs-repo.asciidoc
@@ -0,0 +1,103 @@
+// tag::unix[]
+Linux and macOS installations support Unix-style paths:
+
+[source,yaml]
+----
+path:
+  repo:
+    - /mount/backups
+    - /mount/long_term_backups
+----
+
+After restarting each node, use the <<put-snapshot-repo-api,put snapshot
+repository>> API to register the file system repository. Specify the file
+system's path in `settings.location`:
+
+[source,console]
+----
+PUT /_snapshot/my_fs_backup
+{
+  "type": "fs",
+  "settings": {
+    "location": "/mount/backups/my_fs_backup_location",
+    "compress": true
+  }
+}
+----
+// TEST[skip:no access to path]
+
+If you specify a relative path in `settings.location`, {es} resolves the path
+using the first value in the `path.repo` setting.
+
+[source,console]
+----
+PUT /_snapshot/my_fs_backup
+{
+  "type": "fs",
+  "settings": {
+    "location": "my_fs_backup_location",       <1>
+    "compress": true
+  }
+}
+----
+// TEST[skip:no access to path]
+
+<1> The first value in the `path.repo` setting is `/mount/backups`. This
+relative path, `my_fs_backup_location`, resolves to
+`/mount/backups/my_fs_backup_location`.
+// end::unix[]
+
+
+// tag::win[]
+Windows installations support both DOS and Microsoft UNC paths. Escaped any
+backslashes in the paths. For UNC paths, provide the server and share name as a
+prefix.
+
+[source,yaml]
+----
+path:
+  repo:
+    - "E:\\Mount\\Backups"                      <1>
+    - "\\\\MY_SERVER\\Mount\\Long_term_backups" <2>
+----
+
+<1> DOS path
+<2> UNC path
+
+After restarting each node, use the <<put-snapshot-repo-api,put snapshot
+repository>> API to register the file system repository. Specify the file
+system's path in `settings.location`:
+
+[source,console]
+----
+PUT /_snapshot/my_fs_backup
+{
+  "type": "fs",
+  "settings": {
+    "location": "E:\\Mount\\Backups\\My_fs_backup_location",
+    "compress": true
+  }
+}
+----
+// TEST[skip:no access to path]
+
+If you specify a relative path in `settings.location`, {es} resolves the path
+using the first value in the `path.repo` setting.
+
+[source,console]
+----
+PUT /_snapshot/my_fs_backup
+{
+  "type": "fs",
+  "settings": {
+    "location": "My_fs_backup_location",       <1>
+    "compress": true
+  }
+}
+----
+// TEST[skip:no access to path]
+
+<1> The first value in the `path.repo` setting is `E:\Mount\Backups`. This
+relative path, `My_fs_backup_location`, resolves to
+`E:\Mount\Backups\My_fs_backup_location`.
+// end::win[]
\ No newline at end of file

From 5920baa544e23f91d4445a9cfca4968373989017 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 9 Nov 2020 10:52:38 -0500
Subject: [PATCH 09/34] Default autoscaling to being enabled (#64786)

This commit flips autoscaling to by default being enabled which means
that the various endpoints and infrastructure are registered. We will
follow up in future changes to remove this feature flag but enabling by
default will simplify development.
---
 .../elasticsearch/xpack/autoscaling/Autoscaling.java | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java
index 4d6970d89e980..951365e1a541a 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java
@@ -6,8 +6,6 @@
 
 package org.elasticsearch.xpack.autoscaling;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Build;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -45,12 +43,12 @@
 import org.elasticsearch.xpack.autoscaling.action.TransportGetAutoscalingCapacityAction;
 import org.elasticsearch.xpack.autoscaling.action.TransportGetAutoscalingPolicyAction;
 import org.elasticsearch.xpack.autoscaling.action.TransportPutAutoscalingPolicyAction;
+import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingCalculateCapacityService;
+import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderConfiguration;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult;
+import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderService;
 import org.elasticsearch.xpack.autoscaling.capacity.FixedAutoscalingDeciderConfiguration;
 import org.elasticsearch.xpack.autoscaling.capacity.FixedAutoscalingDeciderService;
-import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderConfiguration;
-import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderService;
-import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingCalculateCapacityService;
 import org.elasticsearch.xpack.autoscaling.rest.RestDeleteAutoscalingPolicyHandler;
 import org.elasticsearch.xpack.autoscaling.rest.RestGetAutoscalingCapacityHandler;
 import org.elasticsearch.xpack.autoscaling.rest.RestGetAutoscalingPolicyHandler;
@@ -68,7 +66,7 @@
  * Container class for autoscaling functionality.
  */
 public class Autoscaling extends Plugin implements ActionPlugin, ExtensiblePlugin, AutoscalingExtension {
-    private static final Logger logger = LogManager.getLogger(AutoscalingExtension.class);
+
     private static final Boolean AUTOSCALING_FEATURE_FLAG_REGISTERED;
 
     static {
@@ -91,7 +89,7 @@ public class Autoscaling extends Plugin implements ActionPlugin, ExtensiblePlugi
 
     public static final Setting<Boolean> AUTOSCALING_ENABLED_SETTING = Setting.boolSetting(
         "xpack.autoscaling.enabled",
-        false,
+        true,
         Setting.Property.NodeScope
     );
 

From b31c235f5857bbeb09bfc569c04ed950e5054fbd Mon Sep 17 00:00:00 2001
From: James Rodewig <40268737+jrodewig@users.noreply.github.com>
Date: Mon, 9 Nov 2020 11:07:36 -0500
Subject: [PATCH 10/34] [master] [DOCS] Remove unneeded period (#64687)
 (#64790)

Co-authored-by: Johannes Mahne <johannes.mahne@elastic.co>
---
 docs/reference/indices/put-component-template.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc
index d6446bec5e5aa..9f8ded246243a 100644
--- a/docs/reference/indices/put-component-template.asciidoc
+++ b/docs/reference/indices/put-component-template.asciidoc
@@ -5,7 +5,7 @@
 ++++
 
 Creates or updates a component template.
-Component templates are building blocks for constructing <<indices-templates,index templates>>.
+Component templates are building blocks for constructing <<indices-templates,index templates>>
 that specify index <<mapping,mappings>>, <<index-modules-settings,settings>>,
 and <<indices-aliases,aliases>>.
 

From 7ed7a9584e32351c457a66cb6b367babc4e612a3 Mon Sep 17 00:00:00 2001
From: James Rodewig <40268737+jrodewig@users.noreply.github.com>
Date: Mon, 9 Nov 2020 11:12:46 -0500
Subject: [PATCH 11/34] [DOCS] Fix typo (#64675) (#64799)

Co-authored-by: Ashish Jayan <58534490+chasexd@users.noreply.github.com>
---
 docs/reference/search/suggesters/phrase-suggest.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/search/suggesters/phrase-suggest.asciidoc b/docs/reference/search/suggesters/phrase-suggest.asciidoc
index c84c6e41cd9a8..e7428b54892f0 100644
--- a/docs/reference/search/suggesters/phrase-suggest.asciidoc
+++ b/docs/reference/search/suggesters/phrase-suggest.asciidoc
@@ -149,7 +149,7 @@ The response contains suggestions scored by the most likely spelling correction
     `gram_size` is set to the `max_shingle_size` if not explicitly set.
 
 `real_word_error_likelihood`::
-    The likelihood of a term being a
+    The likelihood of a term being
     misspelled even if the term exists in the dictionary. The default is
     `0.95`, meaning 5% of the real words are misspelled.
 

From 92e572b583e0bcde136aceb0ec38b06f51f8a0f9 Mon Sep 17 00:00:00 2001
From: Rory Hunter <roryhunter2@gmail.com>
Date: Mon, 9 Nov 2020 16:21:00 +0000
Subject: [PATCH 12/34] Fix PluginInfo BWC

---
 server/src/main/java/org/elasticsearch/plugins/PluginInfo.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java b/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java
index 9ceda83157e10..879e498d1f67f 100644
--- a/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java
+++ b/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java
@@ -50,7 +50,7 @@ public class PluginInfo implements Writeable, ToXContentObject {
     public static final String ES_PLUGIN_PROPERTIES = "plugin-descriptor.properties";
     public static final String ES_PLUGIN_POLICY = "plugin-security.policy";
 
-    private static final Version QUOTA_FS_PLUGIN_SUPPORT = Version.CURRENT;
+    private static final Version QUOTA_FS_PLUGIN_SUPPORT = Version.V_7_11_0;;
 
     private final String name;
     private final String description;

From 505b91efa143212b738dfeaf366ae93e27f0668e Mon Sep 17 00:00:00 2001
From: Rory Hunter <roryhunter2@gmail.com>
Date: Mon, 9 Nov 2020 16:26:28 +0000
Subject: [PATCH 13/34] Fix duplicate semicolon

---
 server/src/main/java/org/elasticsearch/plugins/PluginInfo.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java b/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java
index 879e498d1f67f..b516e903b0ffb 100644
--- a/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java
+++ b/server/src/main/java/org/elasticsearch/plugins/PluginInfo.java
@@ -50,7 +50,7 @@ public class PluginInfo implements Writeable, ToXContentObject {
     public static final String ES_PLUGIN_PROPERTIES = "plugin-descriptor.properties";
     public static final String ES_PLUGIN_POLICY = "plugin-security.policy";
 
-    private static final Version QUOTA_FS_PLUGIN_SUPPORT = Version.V_7_11_0;;
+    private static final Version QUOTA_FS_PLUGIN_SUPPORT = Version.V_7_11_0;
 
     private final String name;
     private final String description;

From e86c65c6c8d62d6bf4773f915471f20a0497f594 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 9 Nov 2020 11:34:40 -0500
Subject: [PATCH 14/34] Prevent some BigArray leaking (#64744)

Its possible for us to fail to give bytes back to the circuit breaker if
we fail building some objects part way. Worse, if the object allocates a
whole page with of memory and *then* fails then it'll keep the memory
allocated forever.

This adds a test utility we can use to assert that it doesn't happen and
then applies it to a bunch of part of the aggregations framework,
catching a few bugs in the process.
---
 .../common/util/BytesRefHash.java             |  18 +++-
 .../elasticsearch/common/util/LongHash.java   |   9 +-
 .../common/util/LongLongHash.java             |   9 +-
 .../common/util/LongObjectPagedHashMap.java   |  13 ++-
 .../common/util/BitArrayTests.java            |  32 ++----
 .../common/util/BytesRefHashTests.java        | 100 ++++++++---------
 .../common/util/LongHashTests.java            | 101 +++++++++---------
 .../common/util/LongLongHashTests.java        |   5 +
 ....java => LongObjectPagedHashMapTests.java} |  11 +-
 .../HyperLogLogPlusPlusSparseTests.java       |  17 ++-
 .../metrics/HyperLogLogPlusPlusTests.java     |  16 ++-
 .../common/util/MockBigArrays.java            |  90 +++++++++++++++-
 12 files changed, 278 insertions(+), 143 deletions(-)
 rename server/src/test/java/org/elasticsearch/common/util/{LongObjectHashMapTests.java => LongObjectPagedHashMapTests.java} (87%)

diff --git a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java
index ee29df51b5573..f58ac762ea078 100644
--- a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java
+++ b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.util;
 
 import com.carrotsearch.hppc.BitMixer;
+
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
@@ -46,10 +47,19 @@ public BytesRefHash(long capacity, BigArrays bigArrays) {
     //Constructor with configurable capacity and load factor.
     public BytesRefHash(long capacity, float maxLoadFactor, BigArrays bigArrays) {
         super(capacity, maxLoadFactor, bigArrays);
-        startOffsets = bigArrays.newLongArray(capacity + 1, false);
-        startOffsets.set(0, 0);
-        bytes = bigArrays.newByteArray(capacity * 3, false);
-        hashes = bigArrays.newIntArray(capacity, false);
+        boolean success = false;
+        try {
+            // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
+            startOffsets = bigArrays.newLongArray(capacity + 1, false);
+            startOffsets.set(0, 0);
+            bytes = bigArrays.newByteArray(capacity * 3, false);
+            hashes = bigArrays.newIntArray(capacity, false);
+            success = true;
+        } finally {
+            if (false == success) {
+                close();
+            }
+        }
         spare = new BytesRef();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/common/util/LongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongHash.java
index 9f19e6cf8ed44..f1894a4665140 100644
--- a/server/src/main/java/org/elasticsearch/common/util/LongHash.java
+++ b/server/src/main/java/org/elasticsearch/common/util/LongHash.java
@@ -41,7 +41,14 @@ public LongHash(long capacity, BigArrays bigArrays) {
     //Constructor with configurable capacity and load factor.
     public LongHash(long capacity, float maxLoadFactor, BigArrays bigArrays) {
         super(capacity, maxLoadFactor, bigArrays);
-        keys = bigArrays.newLongArray(capacity, false);
+        try {
+            // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
+            keys = bigArrays.newLongArray(capacity, false);
+        } finally {
+            if (keys == null) {
+                close();
+            }
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java
index 3ffb98b19aefb..7a2b8d32b5353 100644
--- a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java
+++ b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java
@@ -48,7 +48,14 @@ public LongLongHash(long capacity, BigArrays bigArrays) {
     //Constructor with configurable capacity and load factor.
     public LongLongHash(long capacity, float maxLoadFactor, BigArrays bigArrays) {
         super(capacity, maxLoadFactor, bigArrays);
-        keys = bigArrays.newLongArray(2 * capacity, false);
+        try {
+            // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
+            keys = bigArrays.newLongArray(2 * capacity, false);
+        } finally {
+            if (keys == null) {
+                close();
+            }
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java b/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java
index 42faec680bbf5..4e37f7c88c199 100644
--- a/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java
+++ b/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java
@@ -41,8 +41,17 @@ public LongObjectPagedHashMap(long capacity, BigArrays bigArrays) {
 
     public LongObjectPagedHashMap(long capacity, float maxLoadFactor, BigArrays bigArrays) {
         super(capacity, maxLoadFactor, bigArrays);
-        keys = bigArrays.newLongArray(capacity(), false);
-        values = bigArrays.newObjectArray(capacity());
+        boolean success = false;
+        try {
+            // `super` allocates a big array so we have to `close` if we fail here or we'll leak it.
+            keys = bigArrays.newLongArray(capacity(), false);
+            values = bigArrays.newObjectArray(capacity());
+            success = true;
+        } finally {
+            if (false == success) {
+                close();
+            }
+        }
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java b/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java
index 5dbe2aafedd58..448aa96a695ae 100644
--- a/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/BitArrayTests.java
@@ -19,12 +19,9 @@
 
 package org.elasticsearch.common.util;
 
-import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.breaker.CircuitBreakingException;
-import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -33,8 +30,6 @@
 
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.junit.Assume.assumeThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class BitArrayTests extends ESTestCase {
 
@@ -89,32 +84,17 @@ public void testTooBigIsNotSet() {
     }
 
     public void testClearingDoesntAllocate() {
-        CircuitBreakerService breaker = mock(CircuitBreakerService.class);
         ByteSizeValue max = new ByteSizeValue(1, ByteSizeUnit.KB);
-        when(breaker.getBreaker(CircuitBreaker.REQUEST)).thenReturn(new NoopCircuitBreaker(CircuitBreaker.REQUEST) {
-            private long total = 0;
-
-            @Override
-            public double addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException {
-                total += bytes;
-                if (total > max.getBytes()) {
-                    throw new CircuitBreakingException("test error", bytes, max.getBytes(), Durability.TRANSIENT);
-                }
-                return total;
-            }
-
-            @Override
-            public long addWithoutBreaking(long bytes) {
-                total += bytes;
-                return total;
-            }
-        });
-        BigArrays bigArrays = new BigArrays(null, breaker, CircuitBreaker.REQUEST, true);
+        MockBigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), max);
         try (BitArray bitArray = new BitArray(1, bigArrays)) {
             bitArray.clear(100000000);
         }
     }
 
+    public void testAllocation() {
+        MockBigArrays.assertFitsIn(new ByteSizeValue(100), bigArrays -> new BitArray(1, bigArrays));
+    }
+
     public void testOr() {
         try (BitArray bitArray1 = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE);
              BitArray bitArray2 = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE);
diff --git a/server/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java b/server/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
index b435ca8fd0b8c..d8a9064f3cba7 100644
--- a/server/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
@@ -26,6 +26,7 @@
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 
@@ -36,62 +37,51 @@
 import java.util.Set;
 
 public class BytesRefHashTests extends ESTestCase {
-
-    BytesRefHash hash;
-
-    private BigArrays randomBigArrays() {
+    private BigArrays mockBigArrays() {
         return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
     }
 
-    private void newHash() {
-        if (hash != null) {
-            hash.close();
-        }
+    private BytesRefHash randomHash() {
         // Test high load factors to make sure that collision resolution works fine
         final float maxLoadFactor = 0.6f + randomFloat() * 0.39f;
-        hash = new BytesRefHash(randomIntBetween(0, 100), maxLoadFactor, randomBigArrays());
-    }
-
-    @Override
-    public void setUp() throws Exception {
-        super.setUp();
-        newHash();
+        return new BytesRefHash(randomIntBetween(0, 100), maxLoadFactor, mockBigArrays());
     }
 
     public void testDuel() {
-        final int len = randomIntBetween(1, 100000);
-        final BytesRef[] values = new BytesRef[len];
-        for (int i = 0; i < values.length; ++i) {
-            values[i] = new BytesRef(randomAlphaOfLength(5));
-        }
-        final ObjectLongMap<BytesRef> valueToId = new ObjectLongHashMap<>();
-        final BytesRef[] idToValue = new BytesRef[values.length];
-        final int iters = randomInt(1000000);
-        for (int i = 0; i < iters; ++i) {
-            final BytesRef value = randomFrom(values);
-            if (valueToId.containsKey(value)) {
-                assertEquals(- 1 - valueToId.get(value), hash.add(value, value.hashCode()));
-            } else {
-                assertEquals(valueToId.size(), hash.add(value, value.hashCode()));
-                idToValue[valueToId.size()] = value;
-                valueToId.put(value, valueToId.size());
+        try (BytesRefHash hash = randomHash()) {
+            final int len = randomIntBetween(1, 100000);
+            final BytesRef[] values = new BytesRef[len];
+            for (int i = 0; i < values.length; ++i) {
+                values[i] = new BytesRef(randomAlphaOfLength(5));
+            }
+            final ObjectLongMap<BytesRef> valueToId = new ObjectLongHashMap<>();
+            final BytesRef[] idToValue = new BytesRef[values.length];
+            final int iters = randomInt(1000000);
+            for (int i = 0; i < iters; ++i) {
+                final BytesRef value = randomFrom(values);
+                if (valueToId.containsKey(value)) {
+                    assertEquals(- 1 - valueToId.get(value), hash.add(value, value.hashCode()));
+                } else {
+                    assertEquals(valueToId.size(), hash.add(value, value.hashCode()));
+                    idToValue[valueToId.size()] = value;
+                    valueToId.put(value, valueToId.size());
+                }
             }
-        }
 
-        assertEquals(valueToId.size(), hash.size());
-        for (final ObjectLongCursor<BytesRef> next : valueToId) {
-            assertEquals(next.value, hash.find(next.key, next.key.hashCode()));
-        }
+            assertEquals(valueToId.size(), hash.size());
+            for (final ObjectLongCursor<BytesRef> next : valueToId) {
+                assertEquals(next.value, hash.find(next.key, next.key.hashCode()));
+            }
 
-        for (long i = 0; i < hash.capacity(); ++i) {
-            final long id = hash.id(i);
-            BytesRef spare = new BytesRef();
-            if (id >= 0) {
-                hash.get(id, spare);
-                assertEquals(idToValue[(int) id], spare);
+            for (long i = 0; i < hash.capacity(); ++i) {
+                final long id = hash.id(i);
+                BytesRef spare = new BytesRef();
+                if (id >= 0) {
+                    hash.get(id, spare);
+                    assertEquals(idToValue[(int) id], spare);
+                }
             }
         }
-        hash.close();
     }
 
     // START - tests borrowed from LUCENE
@@ -100,6 +90,7 @@ public void testDuel() {
      * Test method for {@link org.apache.lucene.util.BytesRefHash#size()}.
      */
     public void testSize() {
+        BytesRefHash hash = randomHash();
         BytesRefBuilder ref = new BytesRefBuilder();
         int num = scaledRandomIntBetween(2, 20);
         for (int j = 0; j < num; j++) {
@@ -112,12 +103,14 @@ public void testSize() {
                 ref.copyChars(str);
                 long count = hash.size();
                 long key = hash.add(ref.get());
-                if (key < 0)
+                if (key < 0) {
                     assertEquals(hash.size(), count);
-                else
+                } else {
                     assertEquals(hash.size(), count + 1);
+                }
                 if(i % mod == 0) {
-                    newHash();
+                    hash.close();
+                    hash = randomHash();
                 }
             }
         }
@@ -130,6 +123,7 @@ public void testSize() {
      * .
      */
     public void testGet() {
+        BytesRefHash hash = randomHash();
         BytesRefBuilder ref = new BytesRefBuilder();
         BytesRef scratch = new BytesRef();
         int num = scaledRandomIntBetween(2, 20);
@@ -158,7 +152,8 @@ public void testGet() {
                 ref.copyChars(entry.getKey());
                 assertEquals(ref.get(), hash.get(entry.getValue(), scratch));
             }
-            newHash();
+            hash.close();
+            hash = randomHash();
         }
         hash.close();
     }
@@ -169,6 +164,7 @@ public void testGet() {
      * .
      */
     public void testAdd() {
+        BytesRefHash hash = randomHash();
         BytesRefBuilder ref = new BytesRefBuilder();
         BytesRef scratch = new BytesRef();
         int num = scaledRandomIntBetween(2, 20);
@@ -198,12 +194,14 @@ public void testAdd() {
             }
 
             assertAllIn(strings, hash);
-            newHash();
+            hash.close();
+            hash = randomHash();
         }
         hash.close();
     }
 
     public void testFind() {
+        BytesRefHash hash = randomHash();
         BytesRefBuilder ref = new BytesRefBuilder();
         BytesRef scratch = new BytesRef();
         int num = scaledRandomIntBetween(2, 20);
@@ -233,7 +231,8 @@ public void testFind() {
             }
 
             assertAllIn(strings, hash);
-            newHash();
+            hash.close();
+            hash = randomHash();
         }
         hash.close();
     }
@@ -254,4 +253,7 @@ private void assertAllIn(Set<String> strings, BytesRefHash hash) {
 
     // END - tests borrowed from LUCENE
 
+    public void testAllocation() {
+        MockBigArrays.assertFitsIn(new ByteSizeValue(512), bigArrays -> new BytesRefHash(1, bigArrays));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/util/LongHashTests.java b/server/src/test/java/org/elasticsearch/common/util/LongHashTests.java
index 31b499d5c4984..a733f9b6b8ffc 100644
--- a/server/src/test/java/org/elasticsearch/common/util/LongHashTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/LongHashTests.java
@@ -24,6 +24,7 @@
 import com.carrotsearch.hppc.cursors.LongLongCursor;
 
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 
@@ -34,68 +35,57 @@
 import java.util.Set;
 
 public class LongHashTests extends ESTestCase {
-    LongHash hash;
-
-    private BigArrays randombigArrays() {
+    private BigArrays mockBigArrays() {
         return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
     }
 
-    private void newHash() {
-        if (hash != null) {
-            hash.close();
-        }
-
+    private LongHash randomHash() {
         // Test high load factors to make sure that collision resolution works fine
-        final float maxLoadFactor = 0.6f + randomFloat() * 0.39f;
-        hash = new LongHash(randomIntBetween(0, 100), maxLoadFactor, randombigArrays());
-    }
-
-    @Override
-    public void setUp() throws Exception {
-        super.setUp();
-        newHash();
+        float maxLoadFactor = 0.6f + randomFloat() * 0.39f;
+        return new LongHash(randomIntBetween(0, 100), maxLoadFactor, mockBigArrays());
     }
 
     public void testDuell() {
-        final Long[] values = new Long[randomIntBetween(1, 100000)];
-        for (int i = 0; i < values.length; ++i) {
-            values[i] = randomLong();
-        }
-        final LongLongMap valueToId = new LongLongHashMap();
-        final long[] idToValue = new long[values.length];
-        final int iters = randomInt(1000000);
-        for (int i = 0; i < iters; ++i) {
-            final Long value = randomFrom(values);
-            if (valueToId.containsKey(value)) {
-                assertEquals(-1 - valueToId.get(value), hash.add(value));
-            } else {
-                assertEquals(valueToId.size(), hash.add(value));
-                idToValue[valueToId.size()] = value;
-                valueToId.put(value, valueToId.size());
+        try (LongHash hash = randomHash()) {
+            final Long[] values = new Long[randomIntBetween(1, 100000)];
+            for (int i = 0; i < values.length; ++i) {
+                values[i] = randomLong();
+            }
+            final LongLongMap valueToId = new LongLongHashMap();
+            final long[] idToValue = new long[values.length];
+            final int iters = randomInt(1000000);
+            for (int i = 0; i < iters; ++i) {
+                final Long value = randomFrom(values);
+                if (valueToId.containsKey(value)) {
+                    assertEquals(-1 - valueToId.get(value), hash.add(value));
+                } else {
+                    assertEquals(valueToId.size(), hash.add(value));
+                    idToValue[valueToId.size()] = value;
+                    valueToId.put(value, valueToId.size());
+                }
             }
-        }
 
-        assertEquals(valueToId.size(), hash.size());
-        for (Iterator<LongLongCursor> iterator = valueToId.iterator(); iterator.hasNext(); ) {
-            final LongLongCursor next = iterator.next();
-            assertEquals(next.value, hash.find(next.key));
-        }
+            assertEquals(valueToId.size(), hash.size());
+            for (Iterator<LongLongCursor> iterator = valueToId.iterator(); iterator.hasNext(); ) {
+                final LongLongCursor next = iterator.next();
+                assertEquals(next.value, hash.find(next.key));
+            }
 
-        for (long i = 0; i < hash.capacity(); ++i) {
-            final long id = hash.id(i);
-            if (id >= 0) {
-                assertEquals(idToValue[(int) id], hash.get(id));
+            for (long i = 0; i < hash.capacity(); ++i) {
+                final long id = hash.id(i);
+                if (id >= 0) {
+                    assertEquals(idToValue[(int) id], hash.get(id));
+                }
             }
-        }
 
-        for (long i = 0; i < hash.size(); i++) {
-            assertEquals(idToValue[(int) i], hash.get(i));
+            for (long i = 0; i < hash.size(); i++) {
+                assertEquals(idToValue[(int) i], hash.get(i));
+            }
         }
-
-        hash.close();
     }
 
     public void testSize() {
+        LongHash hash = randomHash();
         int num = scaledRandomIntBetween(2, 20);
         for (int j = 0; j < num; j++) {
             final int mod = 1 + randomInt(40);
@@ -107,7 +97,8 @@ public void testSize() {
                 else
                     assertEquals(hash.size(), count + 1);
                 if (i % mod == 0) {
-                    newHash();
+                    hash.close();
+                    hash = randomHash();
                 }
             }
         }
@@ -115,6 +106,7 @@ public void testSize() {
     }
 
     public void testKey() {
+        LongHash hash = randomHash();
         int num = scaledRandomIntBetween(2, 20);
         for (int j = 0; j < num; j++) {
             Map<Long, Long> longs = new HashMap<>();
@@ -140,12 +132,14 @@ public void testKey() {
                 assertEquals(expected, hash.get(keyIdx));
             }
 
-            newHash();
+            hash.close();
+            hash = randomHash();
         }
         hash.close();
     }
 
     public void testAdd() {
+        LongHash hash = randomHash();
         int num = scaledRandomIntBetween(2, 20);
         for (int j = 0; j < num; j++) {
             Set<Long> longs = new HashSet<>();
@@ -168,12 +162,14 @@ public void testAdd() {
             }
 
             assertAllIn(longs, hash);
-            newHash();
+            hash.close();
+            hash = randomHash();
         }
         hash.close();
     }
 
     public void testFind() throws Exception {
+        LongHash hash = randomHash();
         int num = scaledRandomIntBetween(2, 20);
         for (int j = 0; j < num; j++) {
             Set<Long> longs = new HashSet<>();
@@ -197,11 +193,16 @@ public void testFind() throws Exception {
             }
 
             assertAllIn(longs, hash);
-            newHash();
+            hash.close();
+            hash = randomHash();
         }
         hash.close();
     }
 
+    public void testAllocation() {
+        MockBigArrays.assertFitsIn(new ByteSizeValue(160), bigArrays -> new LongHash(1, bigArrays));
+    }
+
     private static void assertAllIn(Set<Long> longs, LongHash hash) {
         long count = hash.size();
         for (Long l : longs) {
diff --git a/server/src/test/java/org/elasticsearch/common/util/LongLongHashTests.java b/server/src/test/java/org/elasticsearch/common/util/LongLongHashTests.java
index bdfb6b89ccd4a..9910446cd0fea 100644
--- a/server/src/test/java/org/elasticsearch/common/util/LongLongHashTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/LongLongHashTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.util;
 
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 
@@ -100,6 +101,10 @@ public void testDuel() {
         }
     }
 
+    public void testAllocation() {
+        MockBigArrays.assertFitsIn(new ByteSizeValue(256), bigArrays -> new LongLongHash(1, bigArrays));
+    }
+
     class Key {
         long key1;
         long key2;
diff --git a/server/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java b/server/src/test/java/org/elasticsearch/common/util/LongObjectPagedHashMapTests.java
similarity index 87%
rename from server/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java
rename to server/src/test/java/org/elasticsearch/common/util/LongObjectPagedHashMapTests.java
index 02605d35cefdc..758df9b1081a6 100644
--- a/server/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/LongObjectPagedHashMapTests.java
@@ -21,19 +21,20 @@
 
 import com.carrotsearch.hppc.LongObjectHashMap;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 
-public class LongObjectHashMapTests extends ESTestCase {
+public class LongObjectPagedHashMapTests extends ESTestCase {
 
-    private BigArrays randomBigArrays() {
+    private BigArrays mockBigArrays() {
         return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
     }
 
     public void testDuel() {
         final LongObjectHashMap<Object> map1 = new LongObjectHashMap<>();
         final LongObjectPagedHashMap<Object> map2 =
-            new LongObjectPagedHashMap<>(randomInt(42), 0.6f + randomFloat() * 0.39f, randomBigArrays());
+            new LongObjectPagedHashMap<>(randomInt(42), 0.6f + randomFloat() * 0.39f, mockBigArrays());
         final int maxKey = randomIntBetween(1, 10000);
         final int iters = scaledRandomIntBetween(10000, 100000);
         for (int i = 0; i < iters; ++i) {
@@ -61,4 +62,8 @@ public void testDuel() {
         assertEquals(map1, copy);
     }
 
+    public void testAllocation() {
+        MockBigArrays.assertFitsIn(new ByteSizeValue(256), bigArrays -> new LongObjectPagedHashMap<Object>(1, bigArrays));
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java
index d15b90bb3783a..9faa6cf33c40a 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java
@@ -20,11 +20,14 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import com.carrotsearch.hppc.BitMixer;
+
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.CoreMatchers;
@@ -32,8 +35,8 @@
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicLong;
 
-import static org.elasticsearch.search.aggregations.metrics.AbstractHyperLogLog.MAX_PRECISION;
-import static org.elasticsearch.search.aggregations.metrics.AbstractHyperLogLog.MIN_PRECISION;
+import static org.elasticsearch.search.aggregations.metrics.AbstractCardinalityAlgorithm.MAX_PRECISION;
+import static org.elasticsearch.search.aggregations.metrics.AbstractCardinalityAlgorithm.MIN_PRECISION;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -119,4 +122,14 @@ public long addWithoutBreaking(long bytes) {
 
         assertThat(total.get(), CoreMatchers.equalTo(0L));
     }
+
+    public void testAllocation() {
+        int precision = between(MIN_PRECISION, MAX_PRECISION);
+        long initialBucketCount = between(0, 100);
+        MockBigArrays.assertFitsIn(
+            ByteSizeValue.ofBytes(initialBucketCount * 16),
+            bigArrays -> new HyperLogLogPlusPlusSparse(precision, bigArrays, initialBucketCount)
+        );
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java
index 06211a10a024a..e3863ad5b038c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java
@@ -21,17 +21,21 @@
 
 import com.carrotsearch.hppc.BitMixer;
 import com.carrotsearch.hppc.IntHashSet;
+
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.concurrent.atomic.AtomicLong;
 
-import static org.elasticsearch.search.aggregations.metrics.AbstractHyperLogLog.MAX_PRECISION;
-import static org.elasticsearch.search.aggregations.metrics.AbstractHyperLogLog.MIN_PRECISION;
+import static org.elasticsearch.search.aggregations.metrics.AbstractCardinalityAlgorithm.MAX_PRECISION;
+import static org.elasticsearch.search.aggregations.metrics.AbstractCardinalityAlgorithm.MIN_PRECISION;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
@@ -183,4 +187,12 @@ public void testRetrieveCardinality() {
         }
     }
 
+    public void testAllocation() {
+        int precision = between(MIN_PRECISION, MAX_PRECISION);
+        long initialBucketCount = between(0, 100);
+        MockBigArrays.assertFitsIn(
+            ByteSizeValue.ofBytes((initialBucketCount << precision) + initialBucketCount * 4 + PageCacheRecycler.PAGE_SIZE_IN_BYTES * 2),
+            bigArrays -> new HyperLogLogPlusPlus(precision, bigArrays, initialBucketCount)
+        );
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
index 284e6a156f7ff..0bb2f05c20035 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
@@ -21,11 +21,20 @@
 
 import com.carrotsearch.randomizedtesting.RandomizedContext;
 import com.carrotsearch.randomizedtesting.SeedUtils;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Accountables;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 
@@ -37,12 +46,50 @@
 import java.util.Random;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
 
 import static org.elasticsearch.test.ESTestCase.assertBusy;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class MockBigArrays extends BigArrays {
+    private static final Logger logger = LogManager.getLogger(MockBigArrays.class);
+
+    /**
+     * Assert that a function returning a {@link Releasable} runs to completion
+     * when allocated a breaker with that breaks when it uses more than {@code max}
+     * bytes <strong>and</strong> that the function doesn't leak any
+     * {@linkplain BigArray}s if it is given a breaker that allows fewer bytes.
+     */
+    public static void assertFitsIn(ByteSizeValue max, Function<BigArrays, Releasable> run) {
+        long maxBytes = 0;
+        long prevLimit = 0;
+        while (true) {
+            ByteSizeValue limit = ByteSizeValue.ofBytes(maxBytes);
+            MockBigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), limit);
+            Releasable r = null;
+            try {
+                r = run.apply(bigArrays);
+            } catch (CircuitBreakingException e) {
+                if (maxBytes >= max.getBytes()) {
+                    throw new AssertionError("required more than " + maxBytes + " bytes");
+                }
+                prevLimit = maxBytes;
+                maxBytes = Math.min(max.getBytes(), maxBytes + Math.max(1, max.getBytes() / 10));
+                continue;
+            }
+            Releasables.close(r);
+            logger.info(
+                "First successfully built using less than {} and more than {}",
+                ByteSizeValue.ofBytes(maxBytes),
+                ByteSizeValue.ofBytes(prevLimit)
+            );
+            return;
+        }
+    }
 
     /**
      * Tracking allocations is useful when debugging a leak but shouldn't be enabled by default as this would also be very costly
@@ -74,6 +121,11 @@ public static void ensureAllArraysAreReleased() throws Exception {
                             exception.addSuppressed((Throwable) cause);
                         }
                     }
+                    if (TRACK_ALLOCATIONS) {
+                        for (Object allocation : masterCopy.values()) {
+                            exception.addSuppressed((Throwable) allocation);
+                        }
+                    }
                     throw exception;
                 }
             }
@@ -84,6 +136,14 @@ public static void ensureAllArraysAreReleased() throws Exception {
     private final PageCacheRecycler recycler;
     private final CircuitBreakerService breakerService;
 
+    /**
+     * Create {@linkplain BigArrays} with a configured limit.
+     */
+    public MockBigArrays(PageCacheRecycler recycler, ByteSizeValue limit) {
+        this(recycler, mock(CircuitBreakerService.class), true);
+        when(breakerService.getBreaker(CircuitBreaker.REQUEST)).thenReturn(new LimitedBreaker(CircuitBreaker.REQUEST, limit));
+    }
+
     public MockBigArrays(PageCacheRecycler recycler, CircuitBreakerService breakerService) {
         this(recycler, breakerService, false);
     }
@@ -263,9 +323,10 @@ private abstract static class AbstractArrayWrapper {
         AbstractArrayWrapper(boolean clearOnResize) {
             this.clearOnResize = clearOnResize;
             this.originalRelease = new AtomicReference<>();
-            ACQUIRED_ARRAYS.put(this,
-                    TRACK_ALLOCATIONS ? new RuntimeException("Unreleased array from test: " + LuceneTestCase.getTestClass().getName())
-                            : Boolean.TRUE);
+            Object marker = TRACK_ALLOCATIONS
+                ? new RuntimeException("Array allocated from test: " + LuceneTestCase.getTestClass().getName())
+                : true;
+            ACQUIRED_ARRAYS.put(this, marker);
         }
 
         protected abstract BigArray getDelegate();
@@ -567,4 +628,27 @@ public Collection<Accountable> getChildResources() {
         }
     }
 
+    private static class LimitedBreaker extends NoopCircuitBreaker {
+        private final AtomicLong used = new AtomicLong();
+        private final ByteSizeValue max;
+
+        LimitedBreaker(String name, ByteSizeValue max) {
+            super(name);
+            this.max = max;
+        }
+
+        @Override
+        public double addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException {
+            long total = used.addAndGet(bytes);
+            if (total > max.getBytes()) {
+                throw new CircuitBreakingException("test error", bytes, max.getBytes(), Durability.TRANSIENT);
+            }
+            return total;
+        }
+
+        @Override
+        public long addWithoutBreaking(long bytes) {
+            return used.addAndGet(bytes);
+        }
+    }
 }

From 658e26dc160477dcee1751a02bb4d3355d8df92e Mon Sep 17 00:00:00 2001
From: Andras Palinkas <andras.palinkas@elastic.co>
Date: Mon, 9 Nov 2020 12:43:53 -0500
Subject: [PATCH 15/34] SQL: Remove the deprecated `AttributeMap(Map)` calls
 (#64664)

Use the `AttributeMap.builder()` instead of the `AttributeMap(Map)` to
construct immutable `AttributeMap`s.

Clean up after the `ctor` deprecation in #63710
---
 .../xpack/ql/expression/AttributeMap.java     | 48 ++++++-------
 .../xpack/ql/expression/AttributeSet.java     |  4 +-
 .../xpack/ql/expression/Expressions.java      |  3 +-
 .../ql/expression/AttributeMapTests.java      | 69 +++++++------------
 .../xpack/sql/optimizer/Optimizer.java        | 10 +--
 .../xpack/sql/plan/logical/Pivot.java         |  6 +-
 .../xpack/sql/planner/QueryFolder.java        | 13 ++--
 .../querydsl/container/QueryContainer.java    |  5 +-
 .../search/SourceGeneratorTests.java          |  9 +--
 .../container/QueryContainerTests.java        |  8 +--
 10 files changed, 68 insertions(+), 107 deletions(-)

diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java
index 3fc545d735d80..aec6102c6d906 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java
@@ -15,7 +15,6 @@
 import java.util.function.BiConsumer;
 import java.util.stream.Stream;
 
-import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static java.util.Collections.unmodifiableCollection;
 import static java.util.Collections.unmodifiableSet;
@@ -141,8 +140,8 @@ public String toString() {
     }
 
     @SuppressWarnings("rawtypes")
-    public static final AttributeMap EMPTY = new AttributeMap<>();
-    
+    private static final AttributeMap EMPTY = new AttributeMap<>();
+
     @SuppressWarnings("unchecked")
     public static final <E> AttributeMap<E> emptyAttributeMap() {
         return EMPTY;
@@ -157,23 +156,6 @@ public AttributeMap() {
         delegate = new LinkedHashMap<>();
     }
 
-    /**
-     * Please use the {@link AttributeMap#builder()} instead.
-     */
-    @Deprecated
-    public AttributeMap(Map<Attribute, E> attr) {
-        if (attr.isEmpty()) {
-            delegate = emptyMap();
-        }
-        else {
-            delegate = new LinkedHashMap<>(attr.size());
-
-            for (Entry<Attribute, E> entry : attr.entrySet()) {
-                delegate.put(new AttributeWrapper(entry.getKey()), entry.getValue());
-            }
-        }
-    }
-
     public AttributeMap(Attribute key, E value) {
         delegate = singletonMap(new AttributeWrapper(key), value);
     }
@@ -377,23 +359,41 @@ public static <E> Builder<E> builder() {
         return new Builder<>();
     }
 
+    public static <E> Builder<E> builder(AttributeMap<E> map) {
+        return new Builder<E>().putAll(map);
+    }
+
     public static class Builder<E> {
-        private final AttributeMap<E> map = new AttributeMap<>();
+        private AttributeMap<E> map = null;
+        private AttributeMap<E> previouslyBuiltMap = null;
 
         private Builder() {}
 
+        private AttributeMap<E> map() {
+            if (map == null) {
+                map = new AttributeMap<>();
+                if (previouslyBuiltMap != null) {
+                    map.addAll(previouslyBuiltMap);
+                }
+            }
+            return map;
+        }
+
         public Builder<E> put(Attribute attr, E value) {
-            map.add(attr, value);
+            map().add(attr, value);
             return this;
         }
 
         public Builder<E> putAll(AttributeMap<E> m) {
-            map.addAll(m);
+            map().addAll(m);
             return this;
         }
 
         public AttributeMap<E> build() {
-            return new AttributeMap<>(map);
+            AttributeMap<E> m = map();
+            previouslyBuiltMap = m;
+            map = null;
+            return m;
         }
     }
 }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java
index 585d3d5da10c9..3c46f42e80450 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeSet.java
@@ -13,11 +13,9 @@
 import java.util.function.Predicate;
 import java.util.stream.Stream;
 
-import static java.util.Collections.emptyMap;
-
 public class AttributeSet implements Set<Attribute> {
 
-    private static final AttributeMap<Object> EMPTY_DELEGATE = new AttributeMap<>(emptyMap());
+    private static final AttributeMap<Object> EMPTY_DELEGATE = AttributeMap.emptyAttributeMap();
 
     public static final AttributeSet EMPTY = new AttributeSet(EMPTY_DELEGATE);
 
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java
index 3f716d760654b..c5ce1cea5f54b 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java
@@ -21,7 +21,6 @@
 import java.util.function.Predicate;
 
 import static java.util.Collections.emptyList;
-import static java.util.Collections.emptyMap;
 
 public final class Expressions {
 
@@ -65,7 +64,7 @@ public static List<Attribute> asAttributes(List<? extends NamedExpression> named
 
     public static AttributeMap<Expression> asAttributeMap(List<? extends NamedExpression> named) {
         if (named.isEmpty()) {
-            return new AttributeMap<>(emptyMap());
+            return AttributeMap.emptyAttributeMap();
         }
 
         AttributeMap<Expression> map = new AttributeMap<>();
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java
index 41ea7fd44d7ba..298ade4db11da 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.xpack.ql.type.DataTypes;
 
 import java.util.Collection;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -30,12 +29,12 @@ private static Attribute a(String name) {
     }
 
     private static AttributeMap<String> threeMap() {
-        Map<Attribute, String> map = new LinkedHashMap<>();
-        map.put(a("one"), "one");
-        map.put(a("two"), "two");
-        map.put(a("three"), "three");
+        AttributeMap.Builder<String> builder = AttributeMap.builder();
+        builder.put(a("one"), "one");
+        builder.put(a("two"), "two");
+        builder.put(a("three"), "three");
 
-        return new AttributeMap<>(map);
+        return builder.build();
     }
 
     public void testAttributeMapWithSameAliasesCanResolveAttributes() {
@@ -49,19 +48,6 @@ public void testAttributeMapWithSameAliasesCanResolveAttributes() {
         assertTrue(param1.toAttribute().equals(param2.toAttribute()));
         assertFalse(param1.toAttribute().semanticEquals(param2.toAttribute()));
 
-        Map<Attribute, Expression> collectRefs = new LinkedHashMap<>();
-        for (Alias a : List.of(param1, param2)) {
-            collectRefs.put(a.toAttribute(), a.child());
-        }
-        // we can look up the same item by both attributes
-        assertNotNull(collectRefs.get(param1.toAttribute()));
-        assertNotNull(collectRefs.get(param2.toAttribute()));
-        AttributeMap<Expression> attributeMap = new AttributeMap<>(collectRefs);
-
-        // validate that all Alias can be e
-        assertTrue(attributeMap.containsKey(param1.toAttribute()));
-        assertFalse(attributeMap.containsKey(param2.toAttribute())); // results in unknown attribute exception
-
         AttributeMap.Builder<Expression> mapBuilder = AttributeMap.builder();
         for (Alias a : List.of(param1, param2)) {
             mapBuilder.put(a.toAttribute(), a.child());
@@ -69,7 +55,9 @@ public void testAttributeMapWithSameAliasesCanResolveAttributes() {
         AttributeMap<Expression> newAttributeMap = mapBuilder.build();
 
         assertTrue(newAttributeMap.containsKey(param1.toAttribute()));
-        assertTrue(newAttributeMap.containsKey(param2.toAttribute())); // no more unknown attribute exception
+        assertTrue(newAttributeMap.get(param1.toAttribute()) == param1.child());
+        assertTrue(newAttributeMap.containsKey(param2.toAttribute()));
+        assertTrue(newAttributeMap.get(param2.toAttribute()) == param2.child());
     }
 
     private Alias createIntParameterAlias(int index, int value) {
@@ -85,13 +73,13 @@ public void testEmptyConstructor() {
         assertThat(m.isEmpty(), is(true));
     }
 
-    public void testMapConstructor() {
-        Map<Attribute, String> map = new LinkedHashMap<>();
-        map.put(a("one"), "one");
-        map.put(a("two"), "two");
-        map.put(a("three"), "three");
+    public void testBuilder() {
+        AttributeMap.Builder<String> builder = AttributeMap.builder();
+        builder.put(a("one"), "one");
+        builder.put(a("two"), "two");
+        builder.put(a("three"), "three");
 
-        AttributeMap<String> m = new AttributeMap<>(map);
+        AttributeMap<String> m = builder.build();
         assertThat(m.size(), is(3));
         assertThat(m.isEmpty(), is(false));
 
@@ -101,12 +89,16 @@ public void testMapConstructor() {
         assertThat(m.containsValue("one"), is(true));
         assertThat(m.containsValue("on"), is(false));
         assertThat(m.attributeNames(), contains("one", "two", "three"));
-        assertThat(m.values(), contains(map.values().toArray()));
+        assertThat(m.values(), contains("one", "two", "three"));
 
         // defensive copying
-        map.put(a("four"), "four");
+        builder.put(a("four"), "four");
+        AttributeMap<String> m2 = builder.build();
         assertThat(m.size(), is(3));
         assertThat(m.isEmpty(), is(false));
+        assertThat(m2.size(), is(4));
+        assertThat(m.isEmpty(), is(false));
+        assertThat(m2.attributeNames(), contains("one", "two", "three", "four"));
     }
 
     public void testSingleItemConstructor() {
@@ -164,12 +156,7 @@ public void testKeySet() {
         Attribute two = a("two");
         Attribute three = a("three");
 
-        Map<Attribute, String> map = new LinkedHashMap<>();
-        map.put(one, "one");
-        map.put(two, "two");
-        map.put(three, "three");
-
-        Set<Attribute> keySet = new AttributeMap<>(map).keySet();
+        Set<Attribute> keySet = threeMap().keySet();
         assertThat(keySet, contains(one, two, three));
 
         // toObject
@@ -192,12 +179,7 @@ public void testEntrySet() {
         Attribute two = a("two");
         Attribute three = a("three");
 
-        Map<Attribute, String> map = new LinkedHashMap<>();
-        map.put(one, "one");
-        map.put(two, "two");
-        map.put(three, "three");
-
-        Set<Entry<Attribute, String>> set = new AttributeMap<>(map).entrySet();
+        Set<Entry<Attribute, String>> set = threeMap().entrySet();
 
         assertThat(set, hasSize(3));
 
@@ -211,12 +193,9 @@ public void testEntrySet() {
         assertThat(values, contains("one", "two", "three"));
     }
 
-    public void testForEach() {
+    public void testCopy() {
         AttributeMap<String> m = threeMap();
-
-        Map<Attribute, String> collect = new LinkedHashMap<>();
-        m.forEach(collect::put);
-        AttributeMap<String> copy = new AttributeMap<>(collect);
+        AttributeMap<String> copy = AttributeMap.<String>builder().putAll(m).build();
 
         assertThat(m, is(copy));
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java
index 5744d6fbee43d..be4333d55fb3f 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java
@@ -302,7 +302,7 @@ protected LogicalPlan rule(Project project) {
                 OrderBy ob = (OrderBy) project.child();
 
                 // resolve function references (that maybe hiding the target)
-                final Map<Attribute, Function> collectRefs = new LinkedHashMap<>();
+                AttributeMap.Builder<Function> collectRefs = AttributeMap.builder();
 
                 // collect Attribute sources
                 // only Aliases are interesting since these are the only ones that hide expressions
@@ -316,7 +316,7 @@ protected LogicalPlan rule(Project project) {
                     }
                 }));
 
-                AttributeMap<Function> functions = new AttributeMap<>(collectRefs);
+                AttributeMap<Function> functions = collectRefs.build();
 
                 // track the direct parents
                 Map<String, Order> nestedOrders = new LinkedHashMap<>();
@@ -541,14 +541,14 @@ private List<NamedExpression> combineProjections(List<? extends NamedExpression>
             //TODO: this need rewriting when moving functions of NamedExpression
 
             // collect aliases in the lower list
-            Map<Attribute, NamedExpression> map = new LinkedHashMap<>();
+            AttributeMap.Builder<NamedExpression> aliasesBuilder = AttributeMap.builder();
             for (NamedExpression ne : lower) {
                 if ((ne instanceof Attribute) == false) {
-                    map.put(ne.toAttribute(), ne);
+                    aliasesBuilder.put(ne.toAttribute(), ne);
                 }
             }
 
-            AttributeMap<NamedExpression> aliases = new AttributeMap<>(map);
+            AttributeMap<NamedExpression> aliases = aliasesBuilder.build();
             List<NamedExpression> replaced = new ArrayList<>();
 
             // replace any matching attribute with a lower alias (if there's a match)
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/Pivot.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/Pivot.java
index 55cec8d80a6fd..fb6cb91d03e3e 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/Pivot.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/Pivot.java
@@ -23,9 +23,7 @@
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
 
 import java.util.ArrayList;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 
 import static java.util.Collections.singletonList;
@@ -134,7 +132,7 @@ private AttributeSet valuesOutput() {
     
     public AttributeMap<Literal> valuesToLiterals() {
         AttributeSet outValues = valuesOutput();
-        Map<Attribute, Literal> valuesMap = new LinkedHashMap<>();
+        AttributeMap.Builder<Literal> valuesMap = AttributeMap.builder();
 
         int index = 0;
         // for each attribute, associate its value
@@ -152,7 +150,7 @@ public AttributeMap<Literal> valuesToLiterals() {
             }
         }
 
-        return new AttributeMap<>(valuesMap);
+        return valuesMap.build();
     }
 
     @Override
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java
index 9165e1b5d035c..86da3948eaf03 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java
@@ -421,20 +421,21 @@ static EsQueryExec fold(AggregateExec a, EsQueryExec exec) {
 
             // track aliases defined in the SELECT and used inside GROUP BY
             // SELECT x AS a ... GROUP BY a
-            Map<Attribute, Expression> aliasMap = new LinkedHashMap<>();
             String id = null;
+
+            AttributeMap.Builder<Expression> aliases = AttributeMap.builder();
             for (NamedExpression ne : a.aggregates()) {
                 if (ne instanceof Alias) {
-                    aliasMap.put(ne.toAttribute(), ((Alias) ne).child());
+                    aliases.put(ne.toAttribute(), ((Alias) ne).child());
                 }
             }
 
-            if (aliasMap.isEmpty() == false) {
-                Map<Attribute, Expression> newAliases = new LinkedHashMap<>(queryC.aliases());
-                newAliases.putAll(aliasMap);
-                queryC = queryC.withAliases(new AttributeMap<>(newAliases));
+            if (aliases.build().isEmpty() == false) {
+                aliases.putAll(queryC.aliases());
+                queryC = queryC.withAliases(aliases.build());
             }
 
+
             // build the group aggregation
             // NB: any reference in grouping is already "optimized" by its source so there's no need to look for aliases
             GroupingContext groupingContext = groupBy(a.groupings());
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java
index a2761db533f93..64e4245b557ce 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java
@@ -432,9 +432,8 @@ public FieldExtraction resolve(Attribute attribute) {
 
         // update proc (if needed)
         if (qContainer.scalarFunctions().size() != scalarFunctions.size()) {
-            Map<Attribute, Pipe> procs = new LinkedHashMap<>(qContainer.scalarFunctions());
-            procs.put(attr, proc);
-            qContainer = qContainer.withScalarProcessors(new AttributeMap<>(procs));
+            qContainer = qContainer.withScalarProcessors(
+                AttributeMap.builder(qContainer.scalarFunctions).put(attr, proc).build());
         }
 
         return new Tuple<>(qContainer, new ComputedRef(proc));
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java
index 6bee4b338659b..3ef98e6fff235 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SourceGeneratorTests.java
@@ -14,9 +14,7 @@
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.ql.expression.Attribute;
 import org.elasticsearch.xpack.ql.expression.AttributeMap;
-import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.ql.querydsl.container.AttributeSort;
@@ -32,9 +30,6 @@
 import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer;
 import org.elasticsearch.xpack.sql.querydsl.container.ScoreSort;
 
-import java.util.LinkedHashMap;
-import java.util.Map;
-
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
@@ -89,9 +84,7 @@ public void testSortNoneSpecified() {
     public void testSelectScoreForcesTrackingScore() {
         Score score = new Score(Source.EMPTY);
         ReferenceAttribute attr = new ReferenceAttribute(score.source(), "score", score.dataType());
-        Map<Attribute, Expression> alias = new LinkedHashMap<>();
-        alias.put(attr, score);
-        QueryContainer container = new QueryContainer().withAliases(new AttributeMap<>(alias)).addColumn(attr);
+        QueryContainer container = new QueryContainer().withAliases(new AttributeMap<>(attr, score)).addColumn(attr);
         SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(container, null, randomIntBetween(1, 10));
         assertTrue(sourceBuilder.trackScores());
     }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java
index 8886807ad5840..8b7cc542d93b9 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.xpack.ql.expression.Alias;
 import org.elasticsearch.xpack.ql.expression.Attribute;
 import org.elasticsearch.xpack.ql.expression.AttributeMap;
-import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.querydsl.query.BoolQuery;
 import org.elasticsearch.xpack.ql.querydsl.query.MatchAll;
@@ -24,8 +23,6 @@
 import java.util.AbstractMap.SimpleImmutableEntry;
 import java.util.Arrays;
 import java.util.BitSet;
-import java.util.LinkedHashMap;
-import java.util.Map;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
@@ -86,11 +83,8 @@ public void testColumnMaskShouldDuplicateSameAttributes() {
         Attribute fourth = new FieldAttribute(Source.EMPTY, "fourth", esField);
         Alias firstAliased = new Alias(Source.EMPTY, "firstAliased", first);
 
-        Map<Attribute, Expression> aliasesMap = new LinkedHashMap<>();
-        aliasesMap.put(firstAliased.toAttribute(), first);
-
         QueryContainer queryContainer = new QueryContainer()
-            .withAliases(new AttributeMap<>(aliasesMap))
+            .withAliases(new AttributeMap<>(firstAliased.toAttribute(), first))
             .addColumn(third)
             .addColumn(first)
             .addColumn(fourth)

From 52e6db56db9c1feafc8f0230bb749877b1826dbf Mon Sep 17 00:00:00 2001
From: Adam Locke <adam.locke@elastic.co>
Date: Mon, 9 Nov 2020 13:10:27 -0500
Subject: [PATCH 16/34] Remove typo (#64760) (#64807)

* Consistency in writing style

Removing spaces before and after brackets for consistency.

* Remove typo

Remove one of two consecutive "the"s

Co-authored-by: Johannes Mahne <johannes.mahne@elastic.co>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 x-pack/docs/en/security/authentication/saml-guide.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/docs/en/security/authentication/saml-guide.asciidoc b/x-pack/docs/en/security/authentication/saml-guide.asciidoc
index f2823032118cf..19eefe18b6f04 100644
--- a/x-pack/docs/en/security/authentication/saml-guide.asciidoc
+++ b/x-pack/docs/en/security/authentication/saml-guide.asciidoc
@@ -894,7 +894,7 @@ authentication proxy handling the authentication of end users (more correctly,
 APIs require authentication and the necessary authorization level for the authenticated
 user. For this reason, you must create a Service Account user and assign it a role
 that gives it the `manage_saml` cluster privilege. The use of the `manage_token`
-cluster privilege will be necessary after the authentication takes place, so that the
+cluster privilege will be necessary after the authentication takes place, so that
 the service account user can maintain access in order refresh access tokens on
 behalf of the authenticated users or to subsequently log them out.
 

From b31a8ff244f74bd6850d38c411475c2604f51a4f Mon Sep 17 00:00:00 2001
From: James Rodewig <40268737+jrodewig@users.noreply.github.com>
Date: Mon, 9 Nov 2020 14:20:08 -0500
Subject: [PATCH 17/34] [DOCS] Fix put repository API docs (#64811)

---
 .../apis/put-repo-api.asciidoc                | 33 ++++++++++++++-----
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc
index a1b180980ca3a..1fcd91d6c8c18 100644
--- a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc
@@ -150,11 +150,11 @@ plugins:
 +
 --
 (Required, object)
-Contains settings for the repository. Valid properties for the `settings` object
-depend on the repository type, set using the
-<<put-snapshot-repo-api-request-type,`type`>> parameter.
+Contains settings for the repository.
 
-.Valid `settings` properties for `fs` repositories
+The following `settings` properties are valid for all repository types:
+
+.Properties of `settings`
 [%collapsible%open]
 ====
 `chunk_size`::
@@ -168,11 +168,13 @@ file size).
 If `true`, metadata files, such as index mappings and settings, are compressed
 in snapshots. Data files are not compressed. Defaults to `true`.
 
-`location`::
-(Required, string)
-Location of the shared filesystem used to store and retrieve snapshots. This
-location must be registered in the `path.repo` setting on all master and data
-nodes in the cluster.
+`max_number_of_snapshots`::
+(Optional, integer)
+Maximum number of snapshots the repository can contain. Defaults to `500`.
++
+WARNING: We do not recommend increasing `max_number_of_snapshots`. Larger
+snapshot repositories may degrade master node performance and cause stability
+issues. Instead, delete older snapshots or use multiple repositories.
 
 `max_restore_bytes_per_sec`::
 (Optional, <<byte-units,byte value>>)
@@ -206,6 +208,19 @@ the repository but not create snapshots in it.
 =====
 ====
 
+Other accepted `settings` properties depend on the repository type, set using the
+<<put-snapshot-repo-api-request-type,`type`>> parameter.
+
+.Valid `settings` properties for `fs` repositories
+[%collapsible%open]
+====
+`location`::
+(Required, string)
+Location of the shared filesystem used to store and retrieve snapshots. This
+location must be registered in the `path.repo` setting on all master and data
+nodes in the cluster.
+====
+
 .Valid `settings` properties for `source` repositories
 [%collapsible%open]
 ====

From 7ceed1369dcec5eacd4f023e3b917dea5084b3dc Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 9 Nov 2020 14:20:25 -0500
Subject: [PATCH 18/34] Speed up date_histogram without children (#63643)

This speeds up `date_histogram` aggregations without a parent or
children. This is quite common - it's the aggregation that Kibana's Discover
uses all over the place. Also, we hope to be able to use the same
mechanism to speed aggs with children one day, but that day isn't today.

The kind of speedup we're seeing is fairly substantial in many cases:
```
|                              |                                            |  before |   after |    |
| 90th percentile service time |           date_histogram_calendar_interval | 9266.07 | 1376.13 | ms |
| 90th percentile service time |   date_histogram_calendar_interval_with_tz | 9217.21 | 1372.67 | ms |
| 90th percentile service time |              date_histogram_fixed_interval | 8817.36 | 1312.67 | ms |
| 90th percentile service time |      date_histogram_fixed_interval_with_tz | 8801.71 | 1311.69 | ms | <-- discover's agg
| 90th percentile service time | date_histogram_fixed_interval_with_metrics | 44660.2 | 43789.5 | ms |
```

This uses the work we did in #61467 to precompute the rounding points for
a `date_histogram`. Now, when we know the rounding points we execute the
`date_histogram` as a `range` aggregation. This is nice for two reasons:
1. We can further rewrite the `range` aggregation (see below)
2. We don't need to allocate a hash to convert rounding points
   to ordinals.
3. We can send precise cardinality estimates to sub-aggs.

Points 2 and 3 above are nice, but most of the speed difference comes from
point 1. Specifically, we now look into executing `range` aggregations as
a `filters` aggregation. Normally the `filters` aggregation is quite slow
but when it doesn't have a parent or any children then we can execute it
"filter by filter" which is significantly faster. So fast, in fact, that
it is faster than the original `date_histogram`.

The `range` aggregation is *fairly* careful in how it rewrites, giving up
on the `filters` aggregation if it won't collect "filter by filter" and
falling back to its original execution mechanism.


So an aggregation like this:

```
POST _search
{
  "size": 0,
  "query": {
    "range": {
      "dropoff_datetime": {
        "gte": "2015-01-01 00:00:00",
        "lt": "2016-01-01 00:00:00"
      }
    }
  },
  "aggs": {
    "dropoffs_over_time": {
      "date_histogram": {
        "field": "dropoff_datetime",
        "fixed_interval": "60d",
        "time_zone": "America/New_York"
      }
    }
  }
}
```

is executed like:

```
POST _search
{
  "size": 0,
  "query": {
    "range": {
      "dropoff_datetime": {
        "gte": "2015-01-01 00:00:00",
        "lt": "2016-01-01 00:00:00"
      }
    }
  },
  "aggs": {
    "dropoffs_over_time": {
      "range": {
        "field": "dropoff_datetime",
        "ranges": [
          {"from": 1415250000000, "to": 1420434000000},
          {"from": 1420434000000, "to": 1425618000000},
          {"from": 1425618000000, "to": 1430798400000},
          {"from": 1430798400000, "to": 1435982400000},
          {"from": 1435982400000, "to": 1441166400000},
          {"from": 1441166400000, "to": 1446350400000},
          {"from": 1446350400000, "to": 1451538000000},
          {"from": 1451538000000}
        ]
      }
    }
  }
}
```

Which in turn is executed like this:

```
POST _search
{
  "size": 0,
  "query": {
    "range": {
      "dropoff_datetime": {
        "gte": "2015-01-01 00:00:00",
        "lt": "2016-01-01 00:00:00"
      }
    }
  },
  "aggs": {
    "dropoffs_over_time": {
      "filters": {
        "filters": {
          "1": {"range": {"dropoff_datetime": {"gte": "2014-12-30 00:00:00", "lt": "2015-01-05 05:00:00"}}},
          "2": {"range": {"dropoff_datetime": {"gte": "2015-01-05 05:00:00", "lt": "2015-03-06 05:00:00"}}},
          "3": {"range": {"dropoff_datetime": {"gte": "2015-03-06 00:00:00", "lt": "2015-05-05 00:00:00"}}},
          "4": {"range": {"dropoff_datetime": {"gte": "2015-05-05 00:00:00", "lt": "2015-07-04 00:00:00"}}},
          "5": {"range": {"dropoff_datetime": {"gte": "2015-07-04 00:00:00", "lt": "2015-09-02 00:00:00"}}},
          "6": {"range": {"dropoff_datetime": {"gte": "2015-09-02 00:00:00", "lt": "2015-11-01 00:00:00"}}},
          "7": {"range": {"dropoff_datetime": {"gte": "2015-11-01 00:00:00", "lt": "2015-12-31 00:00:00"}}},
          "8": {"range": {"dropoff_datetime": {"gte": "2015-12-31 00:00:00"}}}
        }
      }
    }
  }
}
```

And *that* is faster because we can execute it "filter by filter".

Finally, notice the `range` query filtering the data. That is required for
the data set that I'm using for testing. The "filter by filter" collection
mechanism for the `filters` agg needs special case handling when the query
is a `range` query and the filter is a `range` query and they are both on
the same field. That special case handling "merges" the range query.
Without it "filter by filter" collection is substantially slower. Its still
quite a bit quicker than the standard `filter` collection, but not nearly
as fast as it could be.
---
 .../test/search.aggregation/10_histogram.yml  |  61 ++-
 .../action/search/TransportSearchIT.java      |   5 +
 .../aggregations/bucket/DateHistogramIT.java  |   2 +-
 .../org/elasticsearch/common/Rounding.java    |  23 +
 .../aggregations/AdaptingAggregator.java      | 130 +++++
 .../search/aggregations/Aggregator.java       |   5 +
 .../search/aggregations/AggregatorBase.java   |   1 +
 .../aggregations/AggregatorFactories.java     |  21 +
 .../bucket/DeferringBucketCollector.java      |   5 +
 .../bucket/filter/FiltersAggregator.java      | 324 ++++++++++--
 .../filter/FiltersAggregatorFactory.java      |  24 +-
 .../bucket/filter/MergedPointRangeQuery.java  | 209 ++++++++
 .../histogram/DateHistogramAggregator.java    | 251 +++++++++-
 .../DateHistogramAggregatorFactory.java       |   4 +-
 .../range/AbstractRangeAggregatorFactory.java |   6 +-
 .../GeoDistanceRangeAggregatorFactory.java    |   2 +-
 .../bucket/range/InternalDateRange.java       |   5 +-
 .../bucket/range/InternalRange.java           |   3 +-
 .../bucket/range/RangeAggregator.java         | 463 +++++++++++++++---
 .../bucket/range/RangeAggregatorSupplier.java |   8 +-
 .../support/ValuesSourceConfig.java           |  16 +-
 .../InternalAggregationProfileTree.java       |  24 +-
 .../aggregation/ProfilingAggregator.java      |   5 +
 .../elasticsearch/common/RoundingTests.java   |  31 ++
 .../aggregations/AdaptingAggregatorTests.java | 146 ++++++
 .../bucket/filter/FiltersAggregatorTests.java |  34 +-
 .../filter/MergedPointRangeQueryTests.java    | 249 ++++++++++
 .../DateHistogramAggregatorTestCase.java      |  21 +-
 .../DateHistogramAggregatorTests.java         |  85 ++++
 .../range/DateRangeAggregatorTests.java       |  85 +++-
 .../bucket/range/RangeAggregatorTests.java    | 152 +++++-
 .../support/CoreValuesSourceTypeTests.java    |   4 +-
 .../support/ValuesSourceConfigTests.java      |  16 +
 .../index/mapper/MapperServiceTestCase.java   |  14 +-
 .../analytics/rate/RateAggregatorTests.java   |  63 ++-
 .../xpack/unsignedlong/UnsignedLongTests.java |   1 +
 36 files changed, 2271 insertions(+), 227 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/AdaptingAggregator.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/AdaptingAggregatorTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQueryTests.java

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
index 2371bd5ef86ca..14a5c63862e94 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
@@ -495,6 +495,58 @@ setup:
                 date:
                   type: date
 
+  - do:
+      bulk:
+        index: test_2
+        refresh: true
+        body:
+            - '{"index": {}}'
+            - '{"date": "2000-01-01"}' # This date is intenationally very far in the past so we end up not being able to use the date_histo -> range -> filters optimization
+            - '{"index": {}}'
+            - '{"date": "2000-01-02"}'
+            - '{"index": {}}'
+            - '{"date": "2016-02-01"}'
+            - '{"index": {}}'
+            - '{"date": "2016-03-01"}'
+
+  - do:
+      search:
+        index: test_2
+        body:
+          size: 0
+          profile: true
+          aggs:
+            histo:
+              date_histogram:
+                field: date
+                calendar_interval: month
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 195 }
+  - match: { aggregations.histo.buckets.0.key_as_string: "2000-01-01T00:00:00.000Z" }
+  - match: { aggregations.histo.buckets.0.doc_count: 2 }
+  - match: { profile.shards.0.aggregations.0.type: DateHistogramAggregator }
+  - match: { profile.shards.0.aggregations.0.description: histo }
+  - match: { profile.shards.0.aggregations.0.breakdown.collect_count: 4 }
+  - match: { profile.shards.0.aggregations.0.debug.total_buckets: 3 }
+
+---
+"date_histogram run as filters profiler":
+  - skip:
+      version: " - 7.99.99"
+      reason:  optimization added in 7.11.0, backport pending
+
+  - do:
+      indices.create:
+          index: test_2
+          body:
+            settings:
+              number_of_replicas: 0
+              number_of_shards: 1
+            mappings:
+              properties:
+                date:
+                  type: date
+
   - do:
       bulk:
         index: test_2
@@ -524,10 +576,13 @@ setup:
   - length: { aggregations.histo.buckets: 3 }
   - match: { aggregations.histo.buckets.0.key_as_string: "2016-01-01T00:00:00.000Z" }
   - match: { aggregations.histo.buckets.0.doc_count: 2 }
-  - match: { profile.shards.0.aggregations.0.type: DateHistogramAggregator }
+  - match: { profile.shards.0.aggregations.0.type: DateHistogramAggregator.FromDateRange }
   - match: { profile.shards.0.aggregations.0.description: histo }
-  - match: { profile.shards.0.aggregations.0.breakdown.collect_count: 4 }
-  - match: { profile.shards.0.aggregations.0.debug.total_buckets: 3 }
+  # ultimately this ends up as a filters agg that uses filter by filter collection which is tracked in build_leaf_collector
+  - match: { profile.shards.0.aggregations.0.breakdown.collect_count: 0 }
+  - match: { profile.shards.0.aggregations.0.debug.delegate: RangeAggregator.FromFilters }
+  - match: { profile.shards.0.aggregations.0.debug.delegate_debug.delegate: FiltersAggregator.FilterByFilter }
+  - match: { profile.shards.0.aggregations.0.debug.delegate_debug.delegate_debug.segments_with_deleted_docs: 0 }
 
 ---
 "histogram with hard bounds":
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
index cf0fecb6375f3..565061fb0d171 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
@@ -590,5 +590,10 @@ public ScoreMode scoreMode() {
 
         @Override
         public void preCollection() throws IOException {}
+
+        @Override
+        public Aggregator[] subAggregators() {
+            throw new UnsupportedOperationException();
+        }
     }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
index 0b6d12d59726a..dd85431a087ee 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
@@ -38,10 +38,10 @@
 import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
-import org.elasticsearch.search.aggregations.bucket.histogram.LongBounds;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram;
+import org.elasticsearch.search.aggregations.bucket.histogram.LongBounds;
 import org.elasticsearch.search.aggregations.metrics.Avg;
 import org.elasticsearch.search.aggregations.metrics.Sum;
 import org.elasticsearch.test.ESIntegTestCase;
diff --git a/server/src/main/java/org/elasticsearch/common/Rounding.java b/server/src/main/java/org/elasticsearch/common/Rounding.java
index 4341d9ca65dd6..081a08a2f4a44 100644
--- a/server/src/main/java/org/elasticsearch/common/Rounding.java
+++ b/server/src/main/java/org/elasticsearch/common/Rounding.java
@@ -291,6 +291,13 @@ public interface Prepared {
          * next rounded value in specified units if possible.
          */
         double roundingSize(long utcMillis, DateTimeUnit timeUnit);
+        /**
+         * If this rounding mechanism precalculates rounding points then
+         * this array stores dates such that each date between each entry.
+         * if the rounding mechanism doesn't precalculate points then this
+         * is {@code null}.
+         */
+        long[] fixedRoundingPoints();
     }
     /**
      * Prepare to round many times.
@@ -435,6 +442,11 @@ protected Prepared maybeUseArray(long minUtcMillis, long maxUtcMillis, int max)
             }
             return new ArrayRounding(values, i, this);
         }
+
+        @Override
+        public long[] fixedRoundingPoints() {
+            return null;
+        }
     }
 
     static class TimeUnitRounding extends Rounding {
@@ -1253,6 +1265,12 @@ public long nextRoundingValue(long utcMillis) {
                 public double roundingSize(long utcMillis, DateTimeUnit timeUnit) {
                     return delegatePrepared.roundingSize(utcMillis, timeUnit);
                 }
+
+                @Override
+                public long[] fixedRoundingPoints() {
+                    // TODO we can likely translate here
+                    return null;
+                }
             };
         }
 
@@ -1335,5 +1353,10 @@ public long nextRoundingValue(long utcMillis) {
         public double roundingSize(long utcMillis, DateTimeUnit timeUnit) {
             return delegate.roundingSize(utcMillis, timeUnit);
         }
+
+        @Override
+        public long[] fixedRoundingPoints() {
+            return Arrays.copyOf(values, max);
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AdaptingAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/AdaptingAggregator.java
new file mode 100644
index 0000000000000..832ecce4ed7eb
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AdaptingAggregator.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.ScoreMode;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.search.profile.aggregation.InternalAggregationProfileTree;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.BiConsumer;
+
+/**
+ * An {@linkplain Aggregator} that delegates collection to another
+ * {@linkplain Aggregator} and then translates its results into the results
+ * you'd expect from another aggregation.
+ */
+public abstract class AdaptingAggregator extends Aggregator {
+    private final Aggregator parent;
+    private final Aggregator delegate;
+
+    public AdaptingAggregator(
+        Aggregator parent,
+        AggregatorFactories subAggregators,
+        CheckedFunction<AggregatorFactories, Aggregator, IOException> delegate
+    ) throws IOException {
+        // Its important we set parent first or else when we build the sub-aggregators they can fail because they'll call this.parent.
+        this.parent = parent;
+        /*
+         * Lock the parent of the sub-aggregators to *this* instead of to
+         * the delegate. This keeps the parent link shaped like the requested
+         * agg tree. Thisis how it has always been and some aggs rely on it.
+         */
+        this.delegate = delegate.apply(subAggregators.fixParent(this));
+        assert this.delegate.parent() == parent : "invalid parent set on delegate";
+    }
+
+    /**
+     * Adapt the result from the collecting {@linkplain Aggregator} into the
+     * result expected by this {@linkplain Aggregator}.
+     */
+    protected abstract InternalAggregation adapt(InternalAggregation delegateResult);
+
+    @Override
+    public final void close() {
+        delegate.close();
+    }
+
+    @Override
+    public final ScoreMode scoreMode() {
+        return delegate.scoreMode();
+    }
+
+    @Override
+    public final String name() {
+        return delegate.name();
+    }
+
+    @Override
+    public final Aggregator parent() {
+        return parent;
+    }
+
+    @Override
+    public final Aggregator subAggregator(String name) {
+        return delegate.subAggregator(name);
+    }
+
+    @Override
+    public final LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException {
+        return delegate.getLeafCollector(ctx);
+    }
+
+    @Override
+    public final void preCollection() throws IOException {
+        delegate.preCollection();
+    }
+
+    @Override
+    public final InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
+        InternalAggregation[] delegateResults = delegate.buildAggregations(owningBucketOrds);
+        InternalAggregation[] result = new InternalAggregation[owningBucketOrds.length];
+        for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) {
+            result[ordIdx] = adapt(delegateResults[ordIdx]);
+        }
+        return result;
+    }
+
+    @Override
+    public final InternalAggregation buildEmptyAggregation() {
+        return adapt(delegate.buildEmptyAggregation());
+    }
+
+    @Override
+    public final Aggregator[] subAggregators() {
+        return delegate.subAggregators();
+    }
+
+    @Override
+    public void collectDebugInfo(BiConsumer<String, Object> add) {
+        super.collectDebugInfo(add);
+        add.accept("delegate", InternalAggregationProfileTree.typeFromAggregator(delegate));
+        Map<String, Object> delegateDebug = new HashMap<>();
+        delegate.collectDebugInfo(delegateDebug::put);
+        add.accept("delegate_debug", delegateDebug);
+    }
+
+    public Aggregator delegate() {
+        return delegate;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
index 5e232829aaa7e..5c58b3ac0414b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
@@ -172,6 +172,11 @@ public final InternalAggregation buildTopLevel() throws IOException {
      */
     public void collectDebugInfo(BiConsumer<String, Object> add) {}
 
+    /**
+     * Get the aggregators running under this one.
+     */
+    public abstract Aggregator[] subAggregators();
+
     /** Aggregation mode for sub aggregations. */
     public enum SubAggCollectionMode implements Writeable {
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
index be9c81b5759b9..5cbb3225c17a4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
@@ -224,6 +224,7 @@ public Aggregator parent() {
         return parent;
     }
 
+    @Override
     public Aggregator[] subAggregators() {
         return subAggregators;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
index ffba5ca28fea5..9f9d090515a8a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
@@ -227,6 +227,27 @@ public int countAggregators() {
         return factories.length;
     }
 
+    /**
+     * This returns a copy of {@link AggregatorFactories} modified so that
+     * calls to {@link #createSubAggregators} will ignore the provided parent
+     * aggregator and always use {@code fixedParent} provided in to this
+     * method.
+     * <p>
+     * {@link AdaptingAggregator} uses this to make sure that sub-aggregators
+     * get the {@link AdaptingAggregator} aggregator itself as the parent.
+     */
+    public AggregatorFactories fixParent(Aggregator fixedParent) {
+        AggregatorFactories previous = this;
+        return new AggregatorFactories(factories) {
+            @Override
+            public Aggregator[] createSubAggregators(SearchContext searchContext, Aggregator parent, CardinalityUpperBound cardinality)
+                throws IOException {
+                // Note that we're throwing out the "parent" passed in to this method and using the parent passed to fixParent
+                return previous.createSubAggregators(searchContext, fixedParent, cardinality);
+            }
+        };
+    }
+
     /**
      * A mutable collection of {@link AggregationBuilder}s and
      * {@link PipelineAggregationBuilder}s.
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java
index e75b7f0c308f0..3d0bb2bec1e23 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java
@@ -119,6 +119,11 @@ public Aggregator resolveSortPath(PathElement next, Iterator<PathElement> path)
         public BucketComparator bucketComparator(String key, SortOrder order) {
             throw new UnsupportedOperationException("Can't sort on deferred aggregations");
         }
+
+        @Override
+        public Aggregator[] subAggregators() {
+            return in.subAggregators();
+        }
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java
index 4687ccd323d0b..ff710b99f6f23 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java
@@ -20,6 +20,17 @@
 package org.elasticsearch.search.aggregations.bucket.filter;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BulkScorer;
+import org.apache.lucene.search.CollectionTerminatedException;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.PointRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
 import org.elasticsearch.common.ParseField;
@@ -45,9 +56,16 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.function.Supplier;
+import java.util.function.BiConsumer;
 
-public class FiltersAggregator extends BucketsAggregator {
+/**
+ * Aggregator for {@code filters}. There are two known subclasses,
+ * {@link FilterByFilter} which is fast but only works in some cases and
+ * {@link Compatible} which works in all cases.
+ * {@link FiltersAggregator#build} will build the fastest version that
+ * works with the configuration.
+ */
+public abstract class FiltersAggregator extends BucketsAggregator {
 
     public static final ParseField FILTERS_FIELD = new ParseField("filters");
     public static final ParseField OTHER_BUCKET_FIELD = new ParseField("other_bucket");
@@ -115,58 +133,110 @@ public boolean equals(Object obj) {
         }
     }
 
+    /**
+     * Build an {@link Aggregator} for a {@code filters} aggregation. If there
+     * isn't a parent, there aren't children, and we don't collect "other"
+     * buckets then this will a faster {@link FilterByFilter} aggregator.
+     * Otherwise it'll fall back to a slower aggregator that is
+     * {@link Compatible} with parent, children, and "other" buckets.
+     */
+    public static FiltersAggregator build(
+        String name,
+        AggregatorFactories factories,
+        String[] keys,
+        Query[] filters,
+        boolean keyed,
+        String otherBucketKey,
+        SearchContext context,
+        Aggregator parent,
+        CardinalityUpperBound cardinality,
+        Map<String, Object> metadata
+    ) throws IOException {
+        FiltersAggregator filterOrder = buildFilterOrderOrNull(
+            name,
+            factories,
+            keys,
+            filters,
+            keyed,
+            otherBucketKey,
+            context,
+            parent,
+            cardinality,
+            metadata
+        );
+        if (filterOrder != null) {
+            return filterOrder;
+        }
+        return new FiltersAggregator.Compatible(
+            name,
+            factories,
+            keys,
+            filters,
+            keyed,
+            otherBucketKey,
+            context,
+            parent,
+            cardinality,
+            metadata
+        );
+    }
+
+    /**
+     * Build an {@link Aggregator} for a {@code filters} aggregation if we
+     * can collect {@link FilterByFilter}, otherwise return {@code null}. We can
+     * collect filter by filter if there isn't a parent, there aren't children,
+     * and we don't collect "other" buckets. Collecting {@link FilterByFilter}
+     * is generally going to be much faster than the {@link Compatible} aggregator.
+     */
+    public static FiltersAggregator buildFilterOrderOrNull(
+        String name,
+        AggregatorFactories factories,
+        String[] keys,
+        Query[] filters,
+        boolean keyed,
+        String otherBucketKey,
+        SearchContext context,
+        Aggregator parent,
+        CardinalityUpperBound cardinality,
+        Map<String, Object> metadata
+    ) throws IOException {
+        if (parent != null) {
+            return null;
+        }
+        if (factories.countAggregators() != 0) {
+            return null;
+        }
+        if (otherBucketKey != null) {
+            return null;
+        }
+        return new FiltersAggregator.FilterByFilter(
+            name,
+            keys,
+            filters,
+            keyed,
+            context,
+            parent,
+            cardinality,
+            metadata
+        );
+    }
+
     private final String[] keys;
-    private Supplier<Weight[]> filters;
     private final boolean keyed;
-    private final boolean showOtherBucket;
-    private final String otherBucketKey;
-    private final int totalNumKeys;
+    protected final String otherBucketKey;
 
-    public FiltersAggregator(String name, AggregatorFactories factories, String[] keys, Supplier<Weight[]> filters, boolean keyed,
+    private FiltersAggregator(String name, AggregatorFactories factories, String[] keys, boolean keyed,
             String otherBucketKey, SearchContext context, Aggregator parent, CardinalityUpperBound cardinality,
             Map<String, Object> metadata) throws IOException {
         super(name, factories, context, parent, cardinality.multiply(keys.length + (otherBucketKey == null ? 0 : 1)), metadata);
         this.keyed = keyed;
         this.keys = keys;
-        this.filters = filters;
-        this.showOtherBucket = otherBucketKey != null;
         this.otherBucketKey = otherBucketKey;
-        if (showOtherBucket) {
-            this.totalNumKeys = keys.length + 1;
-        } else {
-            this.totalNumKeys = keys.length;
-        }
-    }
-
-    @Override
-    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
-            final LeafBucketCollector sub) throws IOException {
-        // no need to provide deleted docs to the filter
-        Weight[] filters = this.filters.get();
-        final Bits[] bits = new Bits[filters.length];
-        for (int i = 0; i < filters.length; ++i) {
-            bits[i] = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), filters[i].scorerSupplier(ctx));
-        }
-        return new LeafBucketCollectorBase(sub, null) {
-            @Override
-            public void collect(int doc, long bucket) throws IOException {
-                boolean matched = false;
-                for (int i = 0; i < bits.length; i++) {
-                    if (bits[i].get(doc)) {
-                        collectBucket(sub, doc, bucketOrd(bucket, i));
-                        matched = true;
-                    }
-                }
-                if (showOtherBucket && !matched) {
-                    collectBucket(sub, doc, bucketOrd(bucket, bits.length));
-                }
-            }
-        };
     }
 
     @Override
     public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
-        return buildAggregationsForFixedBucketCount(owningBucketOrds, keys.length + (showOtherBucket ? 1 : 0),
+        return buildAggregationsForFixedBucketCount(owningBucketOrds, keys.length + (otherBucketKey == null ? 0 : 1),
             (offsetInOwningOrd, docCount, subAggregationResults) -> {
                 if (offsetInOwningOrd < keys.length) {
                     return new InternalFilters.InternalBucket(keys[offsetInOwningOrd], docCount,
@@ -185,7 +255,7 @@ public InternalAggregation buildEmptyAggregation() {
             buckets.add(bucket);
         }
 
-        if (showOtherBucket) {
+        if (otherBucketKey != null) {
             InternalFilters.InternalBucket bucket = new InternalFilters.InternalBucket(otherBucketKey, 0, subAggs, keyed);
             buckets.add(bucket);
         }
@@ -193,8 +263,174 @@ public InternalAggregation buildEmptyAggregation() {
         return new InternalFilters(name, buckets, keyed, metadata());
     }
 
-    final long bucketOrd(long owningBucketOrdinal, int filterOrd) {
-        return owningBucketOrdinal * totalNumKeys + filterOrd;
+    /**
+     * Collects results by running each filter against the searcher and doesn't
+     * build any {@link LeafBucketCollector}s which is generally faster than
+     * {@link Compatible} but doesn't support when there is a parent aggregator
+     * or any child aggregators.
+     */
+    private static class FilterByFilter extends FiltersAggregator {
+        private final Query[] filters;
+        private Weight[] filterWeights;
+        private int segmentsWithDeletedDocs;
+
+        FilterByFilter(
+            String name,
+            String[] keys,
+            Query[] filters,
+            boolean keyed,
+            SearchContext context,
+            Aggregator parent,
+            CardinalityUpperBound cardinality,
+            Map<String, Object> metadata
+        ) throws IOException {
+            super(name, AggregatorFactories.EMPTY, keys, keyed, null, context, parent, cardinality, metadata);
+            this.filters = filters;
+        }
+
+        /**
+         * Instead of returning a {@link LeafBucketCollector} we do the
+         * collection ourselves by running the filters directly. This is safe
+         * because we only use this aggregator if there isn't a {@code parent}
+         * which would change how we collect buckets and because we take the
+         * top level query into account when building the filters.
+         */
+        @Override
+        protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
+            if (filterWeights == null) {
+                filterWeights = buildWeights(topLevelQuery(), filters);
+            }
+            Bits live = ctx.reader().getLiveDocs();
+            for (int filterOrd = 0; filterOrd < filters.length; filterOrd++) {
+                BulkScorer scorer = filterWeights[filterOrd].bulkScorer(ctx);
+                if (scorer == null) {
+                    // the filter doesn't match any docs
+                    continue;
+                }
+                TotalHitCountCollector collector = new TotalHitCountCollector();
+                scorer.score(collector, live);
+                incrementBucketDocCount(filterOrd, collector.getTotalHits());
+            }
+            // Throwing this exception is how we communicate to the collection mechanism that we don't need the segment.
+            throw new CollectionTerminatedException();
+        }
+
+        @Override
+        public void collectDebugInfo(BiConsumer<String, Object> add) {
+            super.collectDebugInfo(add);
+            add.accept("segments_with_deleted_docs", segmentsWithDeletedDocs);
+        }
+    }
+
+    /**
+     * Collects results by building a {@link Bits} per filter and testing if
+     * each doc sent to its {@link LeafBucketCollector} is in each filter
+     * which is generally slower than {@link FilterByFilter} but is compatible
+     * with parent and child aggregations.
+     */
+    private static class Compatible extends FiltersAggregator {
+        private final Query[] filters;
+        private Weight[] filterWeights;
+
+        private final int totalNumKeys;
+
+        Compatible(
+            String name,
+            AggregatorFactories factories,
+            String[] keys,
+            Query[] filters,
+            boolean keyed,
+            String otherBucketKey,
+            SearchContext context,
+            Aggregator parent,
+            CardinalityUpperBound cardinality,
+            Map<String, Object> metadata
+        ) throws IOException {
+            super(name, factories, keys, keyed, otherBucketKey, context, parent, cardinality, metadata);
+            this.filters = filters;
+            if (otherBucketKey == null) {
+                this.totalNumKeys = keys.length;
+            } else {
+                this.totalNumKeys = keys.length + 1;
+            }
+        }
+
+        @Override
+        protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
+            if (filterWeights == null) {
+                filterWeights = buildWeights(new MatchAllDocsQuery(), filters);
+            }
+            final Bits[] bits = new Bits[filters.length];
+            for (int i = 0; i < filters.length; ++i) {
+                bits[i] = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), filterWeights[i].scorerSupplier(ctx));
+            }
+            return new LeafBucketCollectorBase(sub, null) {
+                @Override
+                public void collect(int doc, long bucket) throws IOException {
+                    boolean matched = false;
+                    for (int i = 0; i < bits.length; i++) {
+                        if (bits[i].get(doc)) {
+                            collectBucket(sub, doc, bucketOrd(bucket, i));
+                            matched = true;
+                        }
+                    }
+                    if (otherBucketKey != null && false == matched) {
+                        collectBucket(sub, doc, bucketOrd(bucket, bits.length));
+                    }
+                }
+            };
+        }
+
+        final long bucketOrd(long owningBucketOrdinal, int filterOrd) {
+            return owningBucketOrdinal * totalNumKeys + filterOrd;
+        }
+    }
+
+    protected Weight[] buildWeights(Query topLevelQuery, Query filters[]) throws IOException{
+        Weight[] weights = new Weight[filters.length];
+        for (int i = 0; i < filters.length; ++i) {
+            Query filter = filterMatchingBoth(topLevelQuery, filters[i]);
+            weights[i] = searcher().createWeight(searcher().rewrite(filter), ScoreMode.COMPLETE_NO_SCORES, 1);
+        }
+        return weights;
     }
 
+    /**
+     * Make a filter that matches both queries, merging the
+     * {@link PointRangeQuery}s together if possible. The "merging together"
+     * part is provides a fairly substantial speed boost then executing a
+     * top level query on a date and a filter on a date. This kind of thing
+     * is very common when visualizing logs and metrics.
+     */
+    private Query filterMatchingBoth(Query lhs, Query rhs) {
+        if (lhs instanceof MatchAllDocsQuery) {
+            return rhs;
+        }
+        if (rhs instanceof MatchAllDocsQuery) {
+            return lhs;
+        }
+        Query unwrappedLhs = unwrap(lhs);
+        Query unwrappedRhs = unwrap(rhs);
+        if (unwrappedLhs instanceof PointRangeQuery && unwrappedRhs instanceof PointRangeQuery) {
+            Query merged = MergedPointRangeQuery.merge((PointRangeQuery) unwrappedLhs, (PointRangeQuery) unwrappedRhs);
+            if (merged != null) {
+                // Should we rewrap here?
+                return merged;
+            }
+        }
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        builder.add(lhs, BooleanClause.Occur.MUST);
+        builder.add(rhs, BooleanClause.Occur.MUST);
+        return builder.build();
+    }
+
+    private Query unwrap(Query query) {
+        if (query instanceof IndexSortSortedNumericDocValuesRangeQuery) {
+            query = ((IndexSortSortedNumericDocValuesRangeQuery) query).getFallbackQuery();
+        }
+        if (query instanceof IndexOrDocValuesQuery) {
+            query = ((IndexOrDocValuesQuery) query).getIndexQuery();
+        }
+        return query;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java
index 83d49dd12d626..0594286bd77f3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java
@@ -19,11 +19,8 @@
 
 package org.elasticsearch.search.aggregations.bucket.filter;
 
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Weight;
-import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
@@ -66,31 +63,18 @@ public FiltersAggregatorFactory(String name, List<KeyedFilter> filters, boolean
      * necessary. This is done lazily so that the {@link Weight}s are only
      * created if the aggregation collects documents reducing the overhead of
      * the aggregation in the case where no documents are collected.
-     *
-     * Note that as aggregations are initialsed and executed in a serial manner,
+     * <p>
+     * Note that as aggregations are initialized and executed in a serial manner,
      * no concurrency considerations are necessary here.
      */
-    public Weight[] getWeights(SearchContext searchContext) {
-        if (weights == null) {
-            try {
-                IndexSearcher contextSearcher = searchContext.searcher();
-                weights = new Weight[filters.length];
-                for (int i = 0; i < filters.length; ++i) {
-                    this.weights[i] = contextSearcher.createWeight(contextSearcher.rewrite(filters[i]), ScoreMode.COMPLETE_NO_SCORES, 1);
-                }
-            } catch (IOException e) {
-                throw new AggregationInitializationException("Failed to initialse filters for aggregation [" + name() + "]", e);
-            }
-        }
-        return weights;
-    }
+    
 
     @Override
     public Aggregator createInternal(SearchContext searchContext,
                                         Aggregator parent,
                                         CardinalityUpperBound cardinality,
                                         Map<String, Object> metadata) throws IOException {
-        return new FiltersAggregator(name, factories, keys, () -> getWeights(searchContext), keyed,
+        return FiltersAggregator.build(name, factories, keys, filters, keyed,
             otherBucket ? otherBucketKey : null, searchContext, parent, cardinality, metadata);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java
new file mode 100644
index 0000000000000..b21cf396ede81
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.filter;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BulkScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.PointRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
+import org.apache.lucene.search.Weight;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static java.util.Arrays.compareUnsigned;
+
+/**
+ * Query merging two point in range queries.
+ */
+public class MergedPointRangeQuery extends Query {
+    /**
+     * Merge two {@linkplain PointRangeQuery}s into a {@linkplain MergedPointRangeQuery}
+     * that matches points that match both filters.
+     */
+    public static Query merge(PointRangeQuery lhs, PointRangeQuery rhs) {
+        if (lhs.equals(rhs)) {
+            // Lucky case! The queries were the same so their UNION is just the query itself.
+            return lhs;
+        }
+        if (lhs.getField() != rhs.getField() || lhs.getNumDims() != rhs.getNumDims() || lhs.getBytesPerDim() != rhs.getBytesPerDim()) {
+            return null;
+        }
+        return new MergedPointRangeQuery(lhs, rhs);
+    }
+
+    private final String field;
+    private final Query delegateForMultiValuedSegments;
+    private final Query delegateForSingleValuedSegments;
+
+    private MergedPointRangeQuery(PointRangeQuery lhs, PointRangeQuery rhs) {
+        field = lhs.getField();
+        delegateForMultiValuedSegments = new BooleanQuery.Builder().add(lhs, Occur.MUST).add(rhs, Occur.MUST).build();
+        int numDims = lhs.getNumDims();
+        int bytesPerDim = lhs.getBytesPerDim();
+        this.delegateForSingleValuedSegments = pickDelegateForSingleValuedSegments(
+            mergeBound(lhs.getLowerPoint(), rhs.getLowerPoint(), numDims, bytesPerDim, true),
+            mergeBound(lhs.getUpperPoint(), rhs.getUpperPoint(), numDims, bytesPerDim, false),
+            numDims,
+            bytesPerDim
+        );
+    }
+
+    private Query pickDelegateForSingleValuedSegments(byte[] lower, byte[] upper, int numDims, int bytesPerDim) {
+        // If we ended up with disjoint ranges in any dimension then on single valued segments we can't match any docs.
+        for (int dim = 0; dim < numDims; dim++) {
+            int offset = dim * bytesPerDim;
+            if (compareUnsigned(lower, offset, offset + bytesPerDim, upper, offset, offset + bytesPerDim) > 0) {
+                return new MatchNoDocsQuery("disjoint ranges");
+            }
+        }
+        // Otherwise on single valued segments we can only match docs the match the UNION of the two ranges.
+        return new PointRangeQuery(field, lower, upper, numDims) {
+            @Override
+            protected String toString(int dimension, byte[] value) {
+                // Stolen from Lucene's Binary range query. It'd be best to delegate, but the method isn't visible.
+                StringBuilder sb = new StringBuilder();
+                sb.append("(");
+                for (int i = 0; i < value.length; i++) {
+                    if (i > 0) {
+                        sb.append(' ');
+                    }
+                    sb.append(Integer.toHexString(value[i] & 0xFF));
+                }
+                sb.append(')');
+                return sb.toString();
+            }
+        };
+    }
+
+    @Override
+    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+        return new ConstantScoreWeight(this, boost) {
+            Weight multiValuedSegmentWeight;
+            Weight singleValuedSegmentWeight;
+
+            @Override
+            public boolean isCacheable(LeafReaderContext ctx) {
+                return true;
+            }
+
+            @Override
+            public Scorer scorer(LeafReaderContext context) throws IOException {
+                ScorerSupplier scorerSupplier = scorerSupplier(context);
+                if (scorerSupplier == null) {
+                    return null;
+                }
+                return scorerSupplier.get(Long.MAX_VALUE);
+            }
+
+            @Override
+            public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+                /*
+                 * If we're sure docs only have a single value for the field
+                 * we can pick the most compact bounds. If there are multiple values
+                 * for the field we have to run the boolean query.
+                 */
+                PointValues points = context.reader().getPointValues(field);
+                if (points == null) {
+                    return null;
+                }
+                if (points.size() == points.getDocCount()) {
+                    // Each doc that has points has exactly one point.
+                    return singleValuedSegmentWeight().scorerSupplier(context);
+                }
+                return multiValuedSegmentWeight().scorerSupplier(context);
+            }
+
+            @Override
+            public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
+                PointValues points = context.reader().getPointValues(field);
+                if (points == null) {
+                    return null;
+                }
+                if (points.size() == points.getDocCount()) {
+                    // Each doc that has points has exactly one point.
+                    return singleValuedSegmentWeight().bulkScorer(context);
+                }
+                return multiValuedSegmentWeight().bulkScorer(context);
+            }
+
+            private Weight singleValuedSegmentWeight() throws IOException {
+                if (singleValuedSegmentWeight == null) {
+                    singleValuedSegmentWeight = delegateForSingleValuedSegments.createWeight(searcher, scoreMode, boost);
+                }
+                return singleValuedSegmentWeight;
+            }
+
+            private Weight multiValuedSegmentWeight() throws IOException {
+                if (multiValuedSegmentWeight == null) {
+                    multiValuedSegmentWeight = delegateForMultiValuedSegments.createWeight(searcher, scoreMode, boost);
+                }
+                return multiValuedSegmentWeight;
+            }
+        };
+    }
+
+    /**
+     * The query used when we have single valued segments.
+     */
+    Query delegateForSingleValuedSegments() {
+        return delegateForSingleValuedSegments;
+    }
+
+    @Override
+    public String toString(String field) {
+        return "MergedPointRange[" + delegateForMultiValuedSegments.toString(field) + "]";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+        MergedPointRangeQuery other = (MergedPointRangeQuery) obj;
+        return delegateForMultiValuedSegments.equals(other.delegateForMultiValuedSegments)
+            && delegateForSingleValuedSegments.equals(other.delegateForSingleValuedSegments);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(classHash(), delegateForMultiValuedSegments, delegateForSingleValuedSegments);
+    }
+
+    private static byte[] mergeBound(byte[] lhs, byte[] rhs, int numDims, int bytesPerDim, boolean lower) {
+        byte[] merged = new byte[lhs.length];
+        for (int dim = 0; dim < numDims; dim++) {
+            int offset = dim * bytesPerDim;
+            boolean cmp = compareUnsigned(lhs, offset, offset + bytesPerDim, rhs, offset, offset + bytesPerDim) <= 0;
+            byte[] from = (cmp ^ lower) ? lhs : rhs;
+            System.arraycopy(from, offset, merged, offset, bytesPerDim);
+        }
+        return merged;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
index 0ce24385f72b7..125ca3d2c50c6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
@@ -22,35 +22,184 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Rounding;
+import org.elasticsearch.common.Rounding.DateTimeUnit;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AdaptingAggregator;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.CardinalityUpperBound;
 import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
+import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator;
+import org.elasticsearch.search.aggregations.bucket.range.InternalDateRange;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregatorSupplier;
 import org.elasticsearch.search.aggregations.bucket.terms.LongKeyedBucketOrds;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.function.BiConsumer;
 
 /**
- * An aggregator for date values. Every date is rounded down using a configured
- * {@link Rounding}.
- *
- * @see Rounding
+ * Aggregator for {@code date_histogram} that rounds values using
+ * {@link Rounding}. See {@link FromDateRange} which also aggregates for
+ * {@code date_histogram} but does so by running a {@code range} aggregation
+ * over the date and transforming the results. In general
+ * {@link FromDateRange} is faster than {@link DateHistogramAggregator}
+ * but {@linkplain DateHistogramAggregator} works when we can't precalculate
+ * all of the {@link Rounding.Prepared#fixedRoundingPoints() fixed rounding points}.
  */
 class DateHistogramAggregator extends BucketsAggregator implements SizedBucketAggregator {
+    /**
+     * Build an {@link Aggregator} for a {@code date_histogram} aggregation.
+     * If we can determine the bucket boundaries from
+     * {@link Rounding.Prepared#fixedRoundingPoints()} we use
+     * {@link RangeAggregator} to do the actual collecting, otherwise we use
+     * an specialized {@link DateHistogramAggregator Aggregator} specifically
+     * for the {@code date_histogram}s. We prefer to delegate to the
+     * {@linkplain RangeAggregator} because it can sometimes be further
+     * optimized into a {@link FiltersAggregator}. Even when it can't be
+     * optimized, it is going to be marginally faster and consume less memory
+     * than the {@linkplain DateHistogramAggregator} because it doesn't need
+     * to the round points and because it can pass precise cardinality
+     * estimates to its child aggregations.
+     */
+    public static Aggregator build(
+        String name,
+        AggregatorFactories factories,
+        Rounding rounding,
+        BucketOrder order,
+        boolean keyed,
+        long minDocCount,
+        @Nullable LongBounds extendedBounds,
+        @Nullable LongBounds hardBounds,
+        ValuesSourceConfig valuesSourceConfig,
+        SearchContext context,
+        Aggregator parent,
+        CardinalityUpperBound cardinality,
+        Map<String, Object> metadata
+    ) throws IOException {
+        Rounding.Prepared preparedRounding = valuesSourceConfig.roundingPreparer().apply(rounding);
+        Aggregator asRange = adaptIntoRangeOrNull(
+            name,
+            factories,
+            rounding,
+            preparedRounding,
+            order,
+            keyed,
+            minDocCount,
+            extendedBounds,
+            hardBounds,
+            valuesSourceConfig,
+            context,
+            parent,
+            cardinality,
+            metadata
+        );
+        if (asRange != null) {
+            return asRange;
+        }
+        return new DateHistogramAggregator(
+            name,
+            factories,
+            rounding,
+            preparedRounding,
+            order,
+            keyed,
+            minDocCount,
+            extendedBounds,
+            hardBounds,
+            valuesSourceConfig,
+            context,
+            parent,
+            cardinality,
+            metadata
+        );
+    }
+
+    private static FromDateRange adaptIntoRangeOrNull(
+        String name,
+        AggregatorFactories factories,
+        Rounding rounding,
+        Rounding.Prepared preparedRounding,
+        BucketOrder order,
+        boolean keyed,
+        long minDocCount,
+        @Nullable LongBounds extendedBounds,
+        @Nullable LongBounds hardBounds,
+        ValuesSourceConfig valuesSourceConfig,
+        SearchContext context,
+        Aggregator parent,
+        CardinalityUpperBound cardinality,
+        Map<String, Object> metadata
+    ) throws IOException {
+        if (hardBounds != null || extendedBounds != null) {
+            return null;
+        }
+        long[] fixedRoundingPoints = preparedRounding.fixedRoundingPoints();
+        if (fixedRoundingPoints == null) {
+            return null;
+        }
+        // Range aggs use a double to aggregate and we don't want to lose precision.
+        long min = fixedRoundingPoints[0];
+        long max = fixedRoundingPoints[fixedRoundingPoints.length - 1];
+        if (min < -RangeAggregator.MAX_ACCURATE_BOUND || min > RangeAggregator.MAX_ACCURATE_BOUND) {
+            return null;
+        }
+        if (max < -RangeAggregator.MAX_ACCURATE_BOUND || max > RangeAggregator.MAX_ACCURATE_BOUND) {
+            return null;
+        }
+        RangeAggregatorSupplier rangeSupplier = context.getQueryShardContext()
+            .getValuesSourceRegistry()
+            .getAggregator(RangeAggregationBuilder.REGISTRY_KEY, valuesSourceConfig);
+        if (rangeSupplier == null) {
+            return null;
+        }
+        RangeAggregator.Range[] ranges = new RangeAggregator.Range[fixedRoundingPoints.length];
+        for (int i = 0; i < fixedRoundingPoints.length - 1; i++) {
+            ranges[i] = new RangeAggregator.Range(null, (double) fixedRoundingPoints[i], (double) fixedRoundingPoints[i + 1]);
+        }
+        ranges[ranges.length - 1] = new RangeAggregator.Range(null, (double) fixedRoundingPoints[fixedRoundingPoints.length - 1], null);
+        return new DateHistogramAggregator.FromDateRange(
+            parent,
+            factories,
+            subAggregators -> rangeSupplier.build(
+                name,
+                subAggregators,
+                valuesSourceConfig,
+                InternalDateRange.FACTORY,
+                ranges,
+                false,
+                context,
+                parent,
+                cardinality,
+                metadata
+            ),
+            valuesSourceConfig.format(),
+            rounding,
+            preparedRounding,
+            order,
+            minDocCount,
+            extendedBounds,
+            keyed,
+            fixedRoundingPoints
+        );
+    }
 
     private final ValuesSource.Numeric valuesSource;
     private final DocValueFormat formatter;
@@ -72,6 +221,7 @@ class DateHistogramAggregator extends BucketsAggregator implements SizedBucketAg
         String name,
         AggregatorFactories factories,
         Rounding rounding,
+        Rounding.Prepared preparedRounding,
         BucketOrder order,
         boolean keyed,
         long minDocCount,
@@ -86,7 +236,7 @@ class DateHistogramAggregator extends BucketsAggregator implements SizedBucketAg
 
         super(name, factories, aggregationContext, parent, CardinalityUpperBound.MANY, metadata);
         this.rounding = rounding;
-        this.preparedRounding = valuesSourceConfig.roundingPreparer().apply(rounding);
+        this.preparedRounding = preparedRounding;
         this.order = order;
         order.validate(this);
         this.keyed = keyed;
@@ -153,8 +303,6 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I
                 // the contract of the histogram aggregation is that shards must return buckets ordered by key in ascending order
                 CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator());
 
-                // value source will be null for unmapped fields
-                // Important: use `rounding` here, not `shardRounding`
                 InternalDateHistogram.EmptyBucketInfo emptyBucketInfo = minDocCount == 0
                         ? new InternalDateHistogram.EmptyBucketInfo(rounding.withoutOffset(), buildEmptySubAggregations(), extendedBounds)
                         : null;
@@ -195,4 +343,93 @@ public double bucketSize(long bucket, Rounding.DateTimeUnit unitSize) {
             return 1.0;
         }
     }
+
+    static class FromDateRange extends AdaptingAggregator implements SizedBucketAggregator {
+        private final DocValueFormat format;
+        private final Rounding rounding;
+        private final Rounding.Prepared preparedRounding;
+        private final BucketOrder order;
+        private final long minDocCount;
+        private final LongBounds extendedBounds;
+        private final boolean keyed;
+        private final long[] fixedRoundingPoints;
+
+        FromDateRange(
+            Aggregator parent,
+            AggregatorFactories subAggregators,
+            CheckedFunction<AggregatorFactories, Aggregator, IOException> delegate,
+            DocValueFormat format,
+            Rounding rounding,
+            Rounding.Prepared preparedRounding,
+            BucketOrder order,
+            long minDocCount,
+            LongBounds extendedBounds,
+            boolean keyed,
+            long[] fixedRoundingPoints
+        ) throws IOException {
+            super(parent, subAggregators, delegate);
+            this.format = format;
+            this.rounding = rounding;
+            this.preparedRounding = preparedRounding;
+            this.order = order;
+            order.validate(this);
+            this.minDocCount = minDocCount;
+            this.extendedBounds = extendedBounds;
+            this.keyed = keyed;
+            this.fixedRoundingPoints = fixedRoundingPoints;
+        }
+
+        @Override
+        protected InternalAggregation adapt(InternalAggregation delegateResult) {
+            InternalDateRange range = (InternalDateRange) delegateResult;
+            List<InternalDateHistogram.Bucket> buckets = new ArrayList<>(range.getBuckets().size());
+            for (InternalDateRange.Bucket rangeBucket : range.getBuckets()) {
+                if (rangeBucket.getDocCount() > 0) {
+                    buckets.add(
+                        new InternalDateHistogram.Bucket(
+                            rangeBucket.getFrom().toInstant().toEpochMilli(),
+                            rangeBucket.getDocCount(),
+                            keyed,
+                            format,
+                            rangeBucket.getAggregations()
+                        )
+                    );
+                }
+            }
+            CollectionUtil.introSort(buckets, BucketOrder.key(true).comparator());
+
+            InternalDateHistogram.EmptyBucketInfo emptyBucketInfo = minDocCount == 0
+                ? new InternalDateHistogram.EmptyBucketInfo(rounding.withoutOffset(), buildEmptySubAggregations(), extendedBounds)
+                : null;
+            return new InternalDateHistogram(
+                range.getName(),
+                buckets,
+                order,
+                minDocCount,
+                rounding.offset(),
+                emptyBucketInfo,
+                format,
+                keyed,
+                range.getMetadata()
+            );
+        }
+
+        public final InternalAggregations buildEmptySubAggregations() {
+            List<InternalAggregation> aggs = new ArrayList<>();
+            for (Aggregator aggregator : subAggregators()) {
+                aggs.add(aggregator.buildEmptyAggregation());
+            }
+            return InternalAggregations.from(aggs);
+        }
+
+        @Override
+        public double bucketSize(long bucket, DateTimeUnit unitSize) {
+            if (unitSize != null) {
+                long startPoint = bucket < fixedRoundingPoints.length ? fixedRoundingPoints[(int) bucket] : Long.MIN_VALUE;
+                return preparedRounding.roundingSize(startPoint, unitSize);
+            } else {
+                return 1.0;
+            }
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
index a5518c8a3eb4a..d069a7cb8c7a7 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
@@ -42,7 +42,7 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) {
         builder.register(
             DateHistogramAggregationBuilder.REGISTRY_KEY,
             List.of(CoreValuesSourceType.DATE, CoreValuesSourceType.NUMERIC, CoreValuesSourceType.BOOLEAN),
-            DateHistogramAggregator::new,
+            DateHistogramAggregator::build,
                 true);
 
         builder.register(DateHistogramAggregationBuilder.REGISTRY_KEY, CoreValuesSourceType.RANGE, DateRangeHistogramAggregator::new, true);
@@ -111,7 +111,7 @@ protected Aggregator doCreateInternal(
     protected Aggregator createUnmapped(SearchContext searchContext,
                                             Aggregator parent,
                                             Map<String, Object> metadata) throws IOException {
-        return new DateHistogramAggregator(name, factories, rounding, order, keyed, minDocCount, extendedBounds, hardBounds,
+        return new DateHistogramAggregator(name, factories, rounding, null, order, keyed, minDocCount, extendedBounds, hardBounds,
             config, searchContext, parent, CardinalityUpperBound.NONE, metadata);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
index 6790cb5ee9147..1072ad54382ae 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Unmapped;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
-import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
@@ -51,7 +50,7 @@ public static void registerAggregators(
         builder.register(
             registryKey,
             List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN),
-            RangeAggregator::new,
+            RangeAggregator::build,
                 true);
     }
 
@@ -92,8 +91,7 @@ protected Aggregator doCreateInternal(
             .build(
                 name,
                 factories,
-                (Numeric) config.getValuesSource(),
-                config.format(),
+                config,
                 rangeFactory,
                 ranges,
                 keyed,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java
index 790ae59c66b31..8e37af016987e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java
@@ -66,7 +66,7 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) {
                 cardinality,
                 metadata) -> {
                 DistanceSource distanceSource = new DistanceSource((ValuesSource.GeoPoint) valuesSource, distanceType, origin, units);
-                return new RangeAggregator(
+                return RangeAggregator.buildWithoutAttemptedToAdaptToFilters(
                     name,
                     factories,
                     distanceSource,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java
index 2c937ab104c54..fd68e3e1ac4b5 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalDateRange.java
@@ -27,6 +27,7 @@
 import java.io.IOException;
 import java.time.Instant;
 import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
 import java.util.List;
 import java.util.Map;
 
@@ -46,13 +47,13 @@ public Bucket(String key, double from, double to, long docCount, InternalAggrega
         }
 
         @Override
-        public Object getFrom() {
+        public ZonedDateTime getFrom() {
             return Double.isInfinite(((Number) from).doubleValue()) ? null :
                 Instant.ofEpochMilli(((Number) from).longValue()).atZone(ZoneOffset.UTC);
         }
 
         @Override
-        public Object getTo() {
+        public ZonedDateTime getTo() {
             return Double.isInfinite(((Number) to).doubleValue()) ? null :
                 Instant.ofEpochMilli(((Number) to).longValue()).atZone(ZoneOffset.UTC);
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
index c750fdc2d062c..1ba14d7a0715c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
@@ -113,7 +112,7 @@ public long getDocCount() {
         }
 
         @Override
-        public Aggregations getAggregations() {
+        public InternalAggregations getAggregations() {
             return aggregations;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
index 82005aaaf40d4..d28107318e788 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
@@ -19,8 +19,11 @@
 package org.elasticsearch.search.aggregations.bucket.range;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreMode;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -31,7 +34,10 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType;
+import org.elasticsearch.index.mapper.DateFieldMapper.Resolution;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AdaptingAggregator;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.CardinalityUpperBound;
@@ -41,7 +47,12 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
 import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
+import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator;
+import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters;
+import org.elasticsearch.search.aggregations.bucket.range.InternalRange.Factory;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -52,7 +63,21 @@
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
-public class RangeAggregator extends BucketsAggregator {
+/**
+ * Aggregator for {@code range}. There are two known subclasses,
+ * {@link NoOverlap} which is fast but only compatible with ranges that
+ * don't have overlaps and {@link Overlap} which handles overlapping
+ * ranges. There is also {@link FromFilters} which isn't a subclass
+ * but is also a functional aggregator for {@code range}.
+ * {@link RangeAggregator#build} will build the fastest of the three
+ * that is compatible with the requested configuration.
+ */
+public abstract class RangeAggregator extends BucketsAggregator {
+    /**
+     * The maximum {@code long} that can accurately fit into the
+     * {@code double} precision floating point bounds.
+     */
+    public static final long MAX_ACCURATE_BOUND = 1L << 53;
 
     public static final ParseField RANGES_FIELD = new ParseField("ranges");
     public static final ParseField KEYED_FIELD = new ParseField("keyed");
@@ -215,15 +240,198 @@ public boolean equals(Object obj) {
         }
     }
 
-    final ValuesSource.Numeric valuesSource;
-    final DocValueFormat format;
-    final Range[] ranges;
-    final boolean keyed;
-    final InternalRange.Factory rangeFactory;
+    /**
+     * Build an {@link Aggregator} for a {@code range} aggregation. If the
+     * {@code ranges} can be converted into filters then it builds a
+     * {@link FiltersAggregator} and uses that to collect the results
+     * <strong>if</strong> that aggregator can run in "filter by filter"
+     * collection mode. If it can't then we'll collect the ranges using
+     * a native {@link RangeAggregator} which is significantly faster
+     * than the "compatible" collection mechanism for the filters agg.
+     */
+    public static Aggregator build(
+        String name,
+        AggregatorFactories factories,
+        ValuesSourceConfig valuesSourceConfig,
+        InternalRange.Factory<?, ?> rangeFactory,
+        Range[] ranges,
+        boolean keyed,
+        SearchContext context,
+        Aggregator parent,
+        CardinalityUpperBound cardinality,
+        Map<String, Object> metadata
+    ) throws IOException {
+        Aggregator adapted = adaptIntoFiltersOrNull(
+            name,
+            factories,
+            valuesSourceConfig,
+            rangeFactory,
+            ranges,
+            keyed,
+            context,
+            parent,
+            cardinality,
+            metadata
+        );
+        if (adapted != null) {
+            return adapted;
+        }
+        return buildWithoutAttemptedToAdaptToFilters(
+            name,
+            factories,
+            (ValuesSource.Numeric) valuesSourceConfig.getValuesSource(),
+            valuesSourceConfig.format(),
+            rangeFactory,
+            ranges,
+            keyed,
+            context,
+            parent,
+            cardinality,
+            metadata
+        );
+    }
+
+    public static Aggregator adaptIntoFiltersOrNull(
+        String name,
+        AggregatorFactories factories,
+        ValuesSourceConfig valuesSourceConfig,
+        InternalRange.Factory<?, ?> rangeFactory,
+        Range[] ranges,
+        boolean keyed,
+        SearchContext context,
+        Aggregator parent,
+        CardinalityUpperBound cardinality,
+        Map<String, Object> metadata
+    ) throws IOException {
+        if (false == valuesSourceConfig.alignesWithSearchIndex()) {
+            return null;
+        }
+        // TODO bail here for runtime fields. They'll be slower this way. Maybe we can somehow look at the Query?
+        if (valuesSourceConfig.fieldType() instanceof DateFieldType
+            && ((DateFieldType) valuesSourceConfig.fieldType()).resolution() == Resolution.NANOSECONDS) {
+            // We don't generate sensible Queries for nanoseconds.
+            return null;
+        }
+        boolean wholeNumbersOnly = false == ((ValuesSource.Numeric) valuesSourceConfig.getValuesSource()).isFloatingPoint();
+        String[] keys = new String[ranges.length];
+        Query[] filters = new Query[ranges.length];
+        for (int i = 0; i < ranges.length; i++) {
+            /*
+             * If the bounds on the ranges are too high then the `double`s
+             * that we work with will round differently in the native range
+             * aggregator than in the filters aggregator. So we can't use
+             * the filters. That is, if the input data type is a `long` in
+             * the first place. If it isn't then 
+             */
+            if (wholeNumbersOnly && ranges[i].from != Double.NEGATIVE_INFINITY && Math.abs(ranges[i].from) > MAX_ACCURATE_BOUND) {
+                return null;
+            }
+            if (wholeNumbersOnly && ranges[i].to != Double.POSITIVE_INFINITY && Math.abs(ranges[i].to) > MAX_ACCURATE_BOUND) {
+                return null;
+            }
+            keys[i] = Integer.toString(i);
+            /*
+             * Use the native format on the field rather than the one provided
+             * on the valuesSourceConfig because the format on the field is what
+             * we parse. With https://github.com/elastic/elasticsearch/pull/63692
+             * we can just cast to a long here and it'll be taken as millis.
+             */
+            DocValueFormat format = valuesSourceConfig.fieldType().docValueFormat(null, null);
+            // TODO correct the loss of precision from the range somehow.....?
+            filters[i] = valuesSourceConfig.fieldType()
+                .rangeQuery(
+                    ranges[i].from == Double.NEGATIVE_INFINITY ? null : format.format(ranges[i].from),
+                    ranges[i].to == Double.POSITIVE_INFINITY ? null : format.format(ranges[i].to),
+                    true,
+                    false,
+                    ShapeRelation.CONTAINS,
+                    null,
+                    null,
+                    context.getQueryShardContext()
+                );
+        }
+        FiltersAggregator delegate = FiltersAggregator.buildFilterOrderOrNull(
+            name,
+            factories,
+            keys,
+            filters,
+            false,
+            null,
+            context,
+            parent,
+            cardinality,
+            metadata
+        );
+        if (delegate == null) {
+            return null;
+        }
+        RangeAggregator.FromFilters<?> fromFilters = new RangeAggregator.FromFilters<>(
+            parent,
+            factories,
+            subAggregators -> {
+                if (subAggregators.countAggregators() > 0) {
+                    throw new IllegalStateException("didn't expect to have a delegate if there are child aggs");
+                }
+                return delegate;
+            },
+            valuesSourceConfig.format(),
+            ranges,
+            keyed,
+            rangeFactory
+        );
+        return fromFilters;
+    }
+
+    public static Aggregator buildWithoutAttemptedToAdaptToFilters(
+        String name,
+        AggregatorFactories factories,
+        ValuesSource.Numeric valuesSource,
+        DocValueFormat format,
+        InternalRange.Factory<?, ?> rangeFactory,
+        Range[] ranges,
+        boolean keyed,
+        SearchContext context,
+        Aggregator parent,
+        CardinalityUpperBound cardinality,
+        Map<String, Object> metadata
+    ) throws IOException {
+        if (hasOverlap(ranges)) {
+            return new RangeAggregator.Overlap(
+                name,
+                factories,
+                valuesSource,
+                format,
+                rangeFactory,
+                ranges,
+                keyed,
+                context,
+                parent,
+                cardinality,
+                metadata
+            );
+        }
+        return new RangeAggregator.NoOverlap(
+            name,
+            factories,
+            valuesSource,
+            format,
+            rangeFactory,
+            ranges,
+            keyed,
+            context,
+            parent,
+            cardinality,
+            metadata
+        );
+    }
 
-    final double[] maxTo;
+    private final ValuesSource.Numeric valuesSource;
+    private final DocValueFormat format;
+    protected final Range[] ranges;
+    private final boolean keyed;
+    private final InternalRange.Factory rangeFactory;
 
-    public RangeAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format,
+    private RangeAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format,
             InternalRange.Factory rangeFactory, Range[] ranges, boolean keyed, SearchContext context,
             Aggregator parent, CardinalityUpperBound cardinality, Map<String, Object> metadata) throws IOException {
 
@@ -233,15 +441,7 @@ public RangeAggregator(String name, AggregatorFactories factories, ValuesSource.
         this.format = format;
         this.keyed = keyed;
         this.rangeFactory = rangeFactory;
-
         this.ranges = ranges;
-
-        maxTo = new double[this.ranges.length];
-        maxTo[0] = this.ranges[0].to;
-        for (int i = 1; i < this.ranges.length; ++i) {
-            maxTo[i] = Math.max(this.ranges[i].to,maxTo[i-1]);
-        }
-
     }
 
     @Override
@@ -253,8 +453,7 @@ public ScoreMode scoreMode() {
     }
 
     @Override
-    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
-            final LeafBucketCollector sub) throws IOException {
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
         final SortedNumericDoubleValues values = valuesSource.doubleValues(ctx);
         return new LeafBucketCollectorBase(sub, values) {
             @Override
@@ -263,63 +462,14 @@ public void collect(int doc, long bucket) throws IOException {
                     final int valuesCount = values.docValueCount();
                     for (int i = 0, lo = 0; i < valuesCount; ++i) {
                         final double value = values.nextValue();
-                        lo = collect(doc, value, bucket, lo);
+                        lo = RangeAggregator.this.collect(sub, doc, value, bucket, lo);
                     }
                 }
             }
-
-    private int collect(int doc, double value, long owningBucketOrdinal, int lowBound) throws IOException {
-        int lo = lowBound, hi = ranges.length - 1; // all candidates are between these indexes
-        int mid = (lo + hi) >>> 1;
-        while (lo <= hi) {
-            if (value < ranges[mid].from) {
-                hi = mid - 1;
-            } else if (value >= maxTo[mid]) {
-                lo = mid + 1;
-            } else {
-                break;
-            }
-            mid = (lo + hi) >>> 1;
-        }
-        if (lo > hi) return lo; // no potential candidate
-
-        // binary search the lower bound
-        int startLo = lo, startHi = mid;
-        while (startLo <= startHi) {
-            final int startMid = (startLo + startHi) >>> 1;
-            if (value >= maxTo[startMid]) {
-                startLo = startMid + 1;
-            } else {
-                startHi = startMid - 1;
-            }
-        }
-
-        // binary search the upper bound
-        int endLo = mid, endHi = hi;
-        while (endLo <= endHi) {
-            final int endMid = (endLo + endHi) >>> 1;
-            if (value < ranges[endMid].from) {
-                endHi = endMid - 1;
-            } else {
-                endLo = endMid + 1;
-            }
-        }
-
-        assert startLo == lowBound || value >= maxTo[startLo - 1];
-        assert endHi == ranges.length - 1 || value < ranges[endHi + 1].from;
-
-        for (int i = startLo; i <= endHi; ++i) {
-            if (ranges[i].matches(value)) {
-                        collectBucket(sub, doc, subBucketOrdinal(owningBucketOrdinal, i));
-            }
-        }
-
-        return endHi + 1;
-    }
         };
     }
 
-    private long subBucketOrdinal(long owningBucketOrdinal, int rangeOrd) {
+    protected long subBucketOrdinal(long owningBucketOrdinal, int rangeOrd) {
         return owningBucketOrdinal * ranges.length + rangeOrd;
     }
 
@@ -383,4 +533,179 @@ public InternalAggregation buildEmptyAggregation() {
         }
     }
 
+    protected abstract int collect(LeafBucketCollector sub, int doc, double value, long owningBucketOrdinal, int lowBound)
+        throws IOException;
+
+    private static class NoOverlap extends RangeAggregator {
+        NoOverlap(
+            String name,
+            AggregatorFactories factories,
+            Numeric valuesSource,
+            DocValueFormat format,
+            Factory rangeFactory,
+            Range[] ranges,
+            boolean keyed,
+            SearchContext context,
+            Aggregator parent,
+            CardinalityUpperBound cardinality,
+            Map<String, Object> metadata
+        ) throws IOException {
+            super(name, factories, valuesSource, format, rangeFactory, ranges, keyed, context, parent, cardinality, metadata);
+        }
+
+        @Override
+        protected int collect(LeafBucketCollector sub, int doc, double value, long owningBucketOrdinal, int lowBound) throws IOException {
+            int lo = lowBound, hi = ranges.length - 1;
+            while (lo <= hi) {
+                final int mid = (lo + hi) >>> 1;
+                if (value < ranges[mid].from) {
+                    hi = mid - 1;
+                } else if (value >= ranges[mid].to) {
+                    lo = mid + 1;
+                } else {
+                    collectBucket(sub, doc, subBucketOrdinal(owningBucketOrdinal, mid));
+                    // The next value must fall in the next bucket to be collected.
+                    return mid + 1;
+                }
+            }
+            return lo;
+        }
+    }
+
+    private static class Overlap extends RangeAggregator {
+        Overlap(
+            String name,
+            AggregatorFactories factories,
+            Numeric valuesSource,
+            DocValueFormat format,
+            Factory rangeFactory,
+            Range[] ranges,
+            boolean keyed,
+            SearchContext context,
+            Aggregator parent,
+            CardinalityUpperBound cardinality,
+            Map<String, Object> metadata
+        ) throws IOException {
+            super(name, factories, valuesSource, format, rangeFactory, ranges, keyed, context, parent, cardinality, metadata);
+            maxTo = new double[ranges.length];
+            maxTo[0] = ranges[0].to;
+            for (int i = 1; i < ranges.length; ++i) {
+                maxTo[i] = Math.max(ranges[i].to, maxTo[i - 1]);
+            }
+        }
+
+        private final double[] maxTo;
+
+        @Override
+        protected int collect(LeafBucketCollector sub, int doc, double value, long owningBucketOrdinal, int lowBound) throws IOException {
+            int lo = lowBound, hi = ranges.length - 1; // all candidates are between these indexes
+            int mid = (lo + hi) >>> 1;
+            while (lo <= hi) {
+                if (value < ranges[mid].from) {
+                    hi = mid - 1;
+                } else if (value >= maxTo[mid]) {
+                    lo = mid + 1;
+                } else {
+                    break;
+                }
+                mid = (lo + hi) >>> 1;
+            }
+            if (lo > hi) return lo; // no potential candidate
+
+            // binary search the lower bound
+            int startLo = lo, startHi = mid;
+            while (startLo <= startHi) {
+                final int startMid = (startLo + startHi) >>> 1;
+                if (value >= maxTo[startMid]) {
+                    startLo = startMid + 1;
+                } else {
+                    startHi = startMid - 1;
+                }
+            }
+
+            // binary search the upper bound
+            int endLo = mid, endHi = hi;
+            while (endLo <= endHi) {
+                final int endMid = (endLo + endHi) >>> 1;
+                if (value < ranges[endMid].from) {
+                    endHi = endMid - 1;
+                } else {
+                    endLo = endMid + 1;
+                }
+            }
+
+            assert startLo == lowBound || value >= maxTo[startLo - 1];
+            assert endHi == ranges.length - 1 || value < ranges[endHi + 1].from;
+
+            for (int i = startLo; i <= endHi; ++i) {
+                if (ranges[i].matches(value)) {
+                            collectBucket(sub, doc, subBucketOrdinal(owningBucketOrdinal, i));
+                }
+            }
+
+            // The next value must fall in the next bucket to be collected.
+            return endHi + 1;
+        }
+    }
+
+    private static class FromFilters<B extends InternalRange.Bucket> extends AdaptingAggregator {
+        private final DocValueFormat format;
+        private final Range[] ranges;
+        private final boolean keyed;
+        private final InternalRange.Factory<B, ?> rangeFactory;
+
+        FromFilters(
+            Aggregator parent,
+            AggregatorFactories subAggregators,
+            CheckedFunction<AggregatorFactories, Aggregator, IOException> delegate,
+            DocValueFormat format,
+            Range[] ranges,
+            boolean keyed,
+            InternalRange.Factory<B, ?> rangeFactory
+        ) throws IOException {
+            super(parent, subAggregators, delegate);
+            this.format = format;
+            this.ranges = ranges;
+            this.keyed = keyed;
+            this.rangeFactory = rangeFactory;
+        }
+
+        @Override
+        protected InternalAggregation adapt(InternalAggregation delegateResult) {
+            InternalFilters filters = (InternalFilters) delegateResult;
+            if (filters.getBuckets().size() != ranges.length) {
+                throw new IllegalStateException(
+                    "bad number of filters [" + filters.getBuckets().size() + "] expecting [" + ranges.length + "]"
+                );
+            }
+            List<B> buckets = new ArrayList<>(filters.getBuckets().size());
+            for (int i = 0; i < ranges.length; i++) {
+                Range r = ranges[i];
+                InternalFilters.InternalBucket b = filters.getBuckets().get(i);
+                buckets.add(
+                    rangeFactory.createBucket(
+                        r.getKey(),
+                        r.getFrom(),
+                        r.getTo(),
+                        b.getDocCount(),
+                        (InternalAggregations) b.getAggregations(),
+                        keyed,
+                        format
+                    )
+                );
+            }
+            return rangeFactory.create(name(), buckets, format, keyed, filters.getMetadata());
+        }
+    }
+
+    private static boolean hasOverlap(Range[] ranges) {
+        double lastEnd = ranges[0].to;
+        for (int i = 1; i < ranges.length; ++i) {
+            if (ranges[i].from < lastEnd) {
+                return true;
+            }
+            lastEnd = ranges[i].to;
+        }
+        return false;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java
index 4bbfd3050106d..2e6714ee83b3e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorSupplier.java
@@ -18,11 +18,10 @@
  */
 package org.elasticsearch.search.aggregations.bucket.range;
 
-import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.CardinalityUpperBound;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -31,9 +30,8 @@
 public interface RangeAggregatorSupplier {
     Aggregator build(String name,
                      AggregatorFactories factories,
-                     ValuesSource.Numeric valuesSource,
-                     DocValueFormat format,
-                     InternalRange.Factory rangeFactory,
+                     ValuesSourceConfig valuesSourceConfig,
+                     InternalRange.Factory<?, ?> rangeFactory,
                      RangeAggregator.Range[] ranges,
                      boolean keyed,
                      SearchContext context,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
index 5afcd394d645a..bb4386360fc20 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
@@ -385,11 +385,17 @@ public boolean hasGlobalOrdinals() {
      */
     @Nullable
     public Function<byte[], Number> getPointReaderOrNull() {
-        MappedFieldType fieldType = fieldType();
-        if (fieldType != null && script() == null && missing() == null) {
-            return fieldType.pointReaderIfPossible();
-        }
-        return null;
+        return alignesWithSearchIndex() ? fieldType().pointReaderIfPossible() : null;
+    }
+
+    /**
+     * Do {@link ValuesSource}s built by this config line up with the search
+     * index of the underlying field? This'll only return true if the fields
+     * is searchable and there aren't missing values or a script to confuse
+     * the ordering.
+     */
+    public boolean alignesWithSearchIndex() {
+        return script() == null && missing() == null && fieldType() != null && fieldType().isSearchable();
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java b/server/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java
index 44d47ef12245b..5f435c7bc3990 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java
@@ -31,17 +31,7 @@ protected AggregationProfileBreakdown createProfileBreakdown() {
 
     @Override
     protected String getTypeFromElement(Aggregator element) {
-
-        // Anonymous classes (such as NonCollectingAggregator in TermsAgg) won't have a name,
-        // we need to get the super class
-        if (element.getClass().getSimpleName().isEmpty()) {
-            return element.getClass().getSuperclass().getSimpleName();
-        }
-        Class<?> enclosing = element.getClass().getEnclosingClass();
-        if (enclosing != null) {
-            return enclosing.getSimpleName() + "." + element.getClass().getSimpleName();
-        }
-        return element.getClass().getSimpleName();
+        return typeFromAggregator(element);
     }
 
     @Override
@@ -49,4 +39,16 @@ protected String getDescriptionFromElement(Aggregator element) {
         return element.name();
     }
 
+    public static String typeFromAggregator(Aggregator aggregator) {
+        // Anonymous classes (such as NonCollectingAggregator in TermsAgg) won't have a name,
+        // we need to get the super class
+        if (aggregator.getClass().getSimpleName().isEmpty()) {
+            return aggregator.getClass().getSuperclass().getSimpleName();
+        }
+        Class<?> enclosing = aggregator.getClass().getEnclosingClass();
+        if (enclosing != null) {
+            return enclosing.getSimpleName() + "." + aggregator.getClass().getSimpleName();
+        }
+        return aggregator.getClass().getSimpleName();
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java b/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java
index 83023325ac95f..9c88d387c5628 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java
@@ -123,6 +123,11 @@ public String toString() {
         return delegate.toString();
     }
 
+    @Override
+    public Aggregator[] subAggregators() {
+        return delegate.subAggregators();
+    }
+
     public static Aggregator unwrap(Aggregator agg) {
         if (agg instanceof ProfilingAggregator) {
             return ((ProfilingAggregator) agg).delegate;
diff --git a/server/src/test/java/org/elasticsearch/common/RoundingTests.java b/server/src/test/java/org/elasticsearch/common/RoundingTests.java
index fa94cacbbe77c..5e7392caf9fdc 100644
--- a/server/src/test/java/org/elasticsearch/common/RoundingTests.java
+++ b/server/src/test/java/org/elasticsearch/common/RoundingTests.java
@@ -39,9 +39,11 @@
 import java.time.zone.ZoneOffsetTransitionRule;
 import java.time.zone.ZoneRules;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import static java.util.stream.Collectors.toList;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -1017,6 +1019,35 @@ public void testNonMillisecondsBasedUnitCalendarRoundingSize() {
         assertThat(prepared.roundingSize(thirdQuarter, Rounding.DateTimeUnit.HOUR_OF_DAY), closeTo(2208.0, 0.000001));
     }
 
+    public void testFixedRoundingPoints() {
+        Rounding rounding = Rounding.builder(Rounding.DateTimeUnit.QUARTER_OF_YEAR).build();
+        assertFixedRoundingPoints(
+            rounding.prepare(time("2020-01-01T00:00:00"), time("2021-01-01T00:00:00")),
+            "2020-01-01T00:00:00",
+            "2020-04-01T00:00:00",
+            "2020-07-01T00:00:00",
+            "2020-10-01T00:00:00",
+            "2021-01-01T00:00:00"
+        );
+        rounding = Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build();
+        assertFixedRoundingPoints(
+            rounding.prepare(time("2020-01-01T00:00:00"), time("2020-01-06T00:00:00")),
+            "2020-01-01T00:00:00",
+            "2020-01-02T00:00:00",
+            "2020-01-03T00:00:00",
+            "2020-01-04T00:00:00",
+            "2020-01-05T00:00:00",
+            "2020-01-06T00:00:00"
+        );
+    }
+
+    private void assertFixedRoundingPoints(Rounding.Prepared prepared, String... expected) {
+        assertThat(
+            Arrays.stream(prepared.fixedRoundingPoints()).mapToObj(Instant::ofEpochMilli).collect(toList()),
+            equalTo(Arrays.stream(expected).map(RoundingTests::time).map(Instant::ofEpochMilli).collect(toList()))
+        );
+    }
+
     private void assertInterval(long rounded, long nextRoundingValue, Rounding rounding, int minutes,
                                 ZoneId tz) {
         assertInterval(rounded, dateBetween(rounded, nextRoundingValue), nextRoundingValue, rounding, tz);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AdaptingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AdaptingAggregatorTests.java
new file mode 100644
index 0000000000000..fe9bf2c6fe141
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/AdaptingAggregatorTests.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MapperServiceTestCase;
+import org.elasticsearch.search.aggregations.bucket.histogram.SizedBucketAggregator;
+import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class AdaptingAggregatorTests extends MapperServiceTestCase {
+    /**
+     * Its important that sub-aggregations of the {@linkplain AdaptingAggregator}
+     * receive a reference to the {@linkplain AdaptingAggregator} as the parent.
+     * Without it we can't do things like implement {@link SizedBucketAggregator}.
+     */
+    public void testParent() throws IOException {
+        MapperService mapperService = createMapperService(mapping(b -> {}));
+        ValuesSourceRegistry.Builder registry = new ValuesSourceRegistry.Builder();
+        MaxAggregationBuilder.registerAggregators(registry);
+        withAggregationContext(registry.build(), mapperService, List.of(), null, context -> {
+            SearchContext searchContext = mock(SearchContext.class);
+            when(searchContext.bigArrays()).thenReturn(context.bigArrays());
+            AggregatorFactories.Builder sub = AggregatorFactories.builder();
+            sub.addAggregator(new MaxAggregationBuilder("test").field("foo"));
+            AggregatorFactory factory = new DummyAdaptingAggregatorFactory("test", context, null, sub, null);
+            Aggregator adapting = factory.create(searchContext, null, CardinalityUpperBound.ONE);
+            assertThat(adapting.subAggregators()[0].parent(), sameInstance(adapting));
+        });
+    }
+
+    public void testBuildCallsAdapt() throws IOException {
+        MapperService mapperService = createMapperService(mapping(b -> {}));
+        withAggregationContext(mapperService, List.of(), context -> {
+            SearchContext searchContext = mock(SearchContext.class);
+            when(searchContext.bigArrays()).thenReturn(context.bigArrays());
+            AggregatorFactory factory = new DummyAdaptingAggregatorFactory("test", context, null, AggregatorFactories.builder(), null);
+            Aggregator adapting = factory.create(searchContext, null, CardinalityUpperBound.ONE);
+            assertThat(adapting.buildEmptyAggregation().getMetadata(), equalTo(Map.of("dog", "woof")));
+            assertThat(adapting.buildTopLevel().getMetadata(), equalTo(Map.of("dog", "woof")));
+        });
+    }
+
+    private static class DummyAdaptingAggregatorFactory extends AggregatorFactory {
+        DummyAdaptingAggregatorFactory(
+            String name,
+            AggregationContext context,
+            AggregatorFactory parent,
+            AggregatorFactories.Builder subFactoriesBuilder,
+            Map<String, Object> metadata
+        ) throws IOException {
+            super(name, context, parent, subFactoriesBuilder, metadata);
+        }
+
+        @Override
+        protected Aggregator createInternal(
+            SearchContext context,
+            Aggregator parent,
+            CardinalityUpperBound cardinality,
+            Map<String, Object> metadata
+        ) throws IOException {
+            return new DummyAdaptingAggregator(
+                parent,
+                factories,
+                subAggs -> new DummyAggregator(name, subAggs, context, parent, CardinalityUpperBound.ONE, metadata)
+            );
+        }
+    }
+
+    private static class DummyAdaptingAggregator extends AdaptingAggregator {
+        DummyAdaptingAggregator(
+            Aggregator parent,
+            AggregatorFactories subAggregators,
+            CheckedFunction<AggregatorFactories, Aggregator, IOException> delegate
+        ) throws IOException {
+            super(parent, subAggregators, delegate);
+        }
+
+        @Override
+        protected InternalAggregation adapt(InternalAggregation delegateResult) {
+            InternalAggregation result = mock(InternalAggregation.class);
+            when(result.getMetadata()).thenReturn(Map.of("dog", "woof"));
+            return result;
+        }
+    }
+
+    private static class DummyAggregator extends AggregatorBase {
+        protected DummyAggregator(
+            String name,
+            AggregatorFactories factories,
+            SearchContext context,
+            Aggregator parent,
+            CardinalityUpperBound subAggregatorCardinality,
+            Map<String, Object> metadata
+        ) throws IOException {
+            super(name, factories, context, parent, subAggregatorCardinality, metadata);
+        }
+
+        @Override
+        protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
+            // TODO Auto-generated method stub
+            return null;
+        }
+
+        @Override
+        public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException {
+            return new InternalAggregation[] {null};
+        }
+
+        @Override
+        public InternalAggregation buildEmptyAggregation() {
+            // TODO Auto-generated method stub
+            return null;
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java
index 617eec9799a4d..a36fff08e1a4e 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java
@@ -20,24 +20,35 @@
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.DateFieldMapper.Resolution;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
 public class FiltersAggregatorTests extends AggregatorTestCase {
     private MappedFieldType fieldType;
 
@@ -139,7 +150,7 @@ public void testRandom() throws Exception {
                 // make sure we have more than one segment to test the merge
                 indexWriter.commit();
             }
-            int value = randomInt(maxTerm-1);
+            int value = randomInt(maxTerm - 1);
             expectedBucketCount[value] += 1;
             document.add(new Field("field", Integer.toString(value), KeywordFieldMapper.Defaults.FIELD_TYPE));
             indexWriter.addDocument(document);
@@ -188,4 +199,25 @@ public void testRandom() throws Exception {
             directory.close();
         }
     }
+
+    public void testMergePointRangeQueries() throws IOException {
+        MappedFieldType ft = new DateFieldMapper.DateFieldType("test", Resolution.MILLISECONDS);
+        AggregationBuilder builder = new FiltersAggregationBuilder(
+            "test",
+            new KeyedFilter("q1", new RangeQueryBuilder("test").from("2020-01-01").to("2020-03-01").includeUpper(false))
+        );
+        Query query = LongPoint.newRangeQuery(
+            "test",
+            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2020-01-01"),
+            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2020-02-01")
+        );
+        testCase(builder, query, iw -> {
+            iw.addDocument(List.of(new LongPoint("test", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2010-01-02"))));
+            iw.addDocument(List.of(new LongPoint("test", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2020-01-02"))));
+        }, result -> {
+            InternalFilters filters = (InternalFilters) result;
+            assertThat(filters.getBuckets(), hasSize(1));
+            assertThat(filters.getBucketByKey("q1").getDocCount(), equalTo(1L));
+        }, ft);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQueryTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQueryTests.java
new file mode 100644
index 0000000000000..d24e756d422f2
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQueryTests.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.filter;
+
+import org.apache.lucene.document.DoublePoint;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.PointRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
+
+public class MergedPointRangeQueryTests extends ESTestCase {
+    public void testDifferentField() {
+        assertThat(merge(LongPoint.newExactQuery("a", 0), LongPoint.newExactQuery("b", 0)), nullValue());
+    }
+
+    public void testDifferentDimensionCount() {
+        assertThat(
+            merge(LongPoint.newExactQuery("a", 0), LongPoint.newRangeQuery("a", new long[] { 1, 2 }, new long[] { 1, 2 })),
+            nullValue()
+        );
+    }
+
+    public void testDifferentDimensionSize() {
+        assertThat(merge(LongPoint.newExactQuery("a", 0), IntPoint.newExactQuery("a", 0)), nullValue());
+    }
+
+    public void testSame() {
+        Query lhs = LongPoint.newRangeQuery("a", 0, 100);
+        assertThat(merge(lhs, LongPoint.newRangeQuery("a", 0, 100)), equalTo(lhs));
+    }
+
+    public void testOverlap() throws IOException {
+        MergedPointRangeQuery overlapping = mergeToMergedQuery(
+            LongPoint.newRangeQuery("a", -100, 100),
+            LongPoint.newRangeQuery("a", 0, 100)
+        );
+        assertDelegateForSingleValuedSegmentsEqualPointRange(overlapping, LongPoint.newRangeQuery("a", 0, 100));
+        assertFalse(matches1d(overlapping, -50));       // Point not in range
+        assertTrue(matches1d(overlapping, 50));         // Point in range
+        assertTrue(matches1d(overlapping, -50, 10));    // Both points in range matches the doc
+        assertTrue(matches1d(overlapping, -200, 50));   // One point in range matches
+        assertFalse(matches1d(overlapping, -50, 200));  // No points in range doesn't match
+    }
+
+    public void testNonOverlap() throws IOException {
+        MergedPointRangeQuery disjoint = mergeToMergedQuery(LongPoint.newRangeQuery("a", -100, -10), LongPoint.newRangeQuery("a", 10, 100));
+        assertThat(disjoint.delegateForSingleValuedSegments(), instanceOf(MatchNoDocsQuery.class));
+        assertFalse(matches1d(disjoint, randomLong()));   // No single point can match
+        assertFalse(matches1d(disjoint, -50, -20));       // Both points in lower
+        assertFalse(matches1d(disjoint, 20, 50));         // Both points in upper
+        assertTrue(matches1d(disjoint, -50, 50));         // One in lower, one in upper
+        assertFalse(matches1d(disjoint, -50, 200));       // No point in lower
+        assertFalse(matches1d(disjoint, -200, 50));       // No point in upper
+    }
+
+    public void test2dSimpleOverlap() throws IOException {
+        MergedPointRangeQuery overlapping = mergeToMergedQuery(
+            LongPoint.newRangeQuery("a", new long[] { -100, -100 }, new long[] { 100, 100 }),
+            LongPoint.newRangeQuery("a", new long[] { 0, 0 }, new long[] { 100, 100 })
+        );
+        assertDelegateForSingleValuedSegmentsEqualPointRange(
+            overlapping,
+            LongPoint.newRangeQuery("a", new long[] { 0, 0 }, new long[] { 100, 100 })
+        );
+        assertFalse(matches2d(overlapping, -50, -50));
+        assertTrue(matches2d(overlapping, 10, 10));
+        assertTrue(matches2d(overlapping, -50, -50, 10, 10));
+    }
+
+    public void test2dComplexOverlap() throws IOException {
+        MergedPointRangeQuery overlapping = mergeToMergedQuery(
+            LongPoint.newRangeQuery("a", new long[] { -100, 0 }, new long[] { 100, 100 }),
+            LongPoint.newRangeQuery("a", new long[] { 0, -100 }, new long[] { 100, 100 })
+        );
+        assertDelegateForSingleValuedSegmentsEqualPointRange(
+            overlapping,
+            LongPoint.newRangeQuery("a", new long[] { 0, 0 }, new long[] { 100, 100 })
+        );
+        assertFalse(matches2d(overlapping, -50, -50));
+        assertTrue(matches2d(overlapping, 10, 10));
+        assertTrue(matches2d(overlapping, -50, -50, 10, 10));
+    }
+
+    public void test2dNoOverlap() throws IOException {
+        MergedPointRangeQuery disjoint = mergeToMergedQuery(
+            LongPoint.newRangeQuery("a", new long[] { -100, -100 }, new long[] { -10, -10 }),
+            LongPoint.newRangeQuery("a", new long[] { 10, 10 }, new long[] { 100, 100 })
+        );
+        assertThat(disjoint.delegateForSingleValuedSegments(), instanceOf(MatchNoDocsQuery.class));
+        assertFalse(matches2d(disjoint, randomLong(), randomLong()));
+        assertFalse(matches2d(disjoint, -50, -50));
+        assertFalse(matches2d(disjoint, 50, 50));
+        assertTrue(matches2d(disjoint, -50, -50, 50, 50));
+    }
+
+    public void test2dNoOverlapInOneDimension() throws IOException {
+        MergedPointRangeQuery disjoint = mergeToMergedQuery(
+            LongPoint.newRangeQuery("a", new long[] { -100, -100 }, new long[] { 100, -10 }),
+            LongPoint.newRangeQuery("a", new long[] { 0, 10 }, new long[] { 100, 100 })
+        );
+        assertThat(disjoint.delegateForSingleValuedSegments(), instanceOf(MatchNoDocsQuery.class));
+        assertFalse(matches2d(disjoint, randomLong(), randomLong()));
+        assertFalse(matches2d(disjoint, -50, -50));
+        assertFalse(matches2d(disjoint, 50, 50));
+        assertTrue(matches2d(disjoint, 50, -50, 50, 50));
+    }
+
+    public void testEqualsAndHashCode() {
+        String field = randomAlphaOfLength(5);
+        int dims = randomBoolean() ? 1 : between(2, 16);
+        Supplier<Query> supplier = randomFrom(
+            List.of(
+                () -> randomIntPointRangequery(field, dims),
+                () -> randomLongPointRangequery(field, dims),
+                () -> randomDoublePointRangequery(field, dims)
+            )
+        );
+        Query lhs = supplier.get();
+        Query rhs = randomValueOtherThan(lhs, supplier);
+        MergedPointRangeQuery query = mergeToMergedQuery(lhs, rhs);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(
+            query,
+            ignored -> mergeToMergedQuery(lhs, rhs),
+            ignored -> mergeToMergedQuery(lhs, randomValueOtherThan(lhs, () -> randomValueOtherThan(rhs, supplier)))
+        );
+    }
+
+    private Query randomIntPointRangequery(String field, int dims) {
+        int[] lower = new int[dims];
+        int[] upper = new int[dims];
+        for (int i = 0; i < dims; i++) {
+            lower[i] = randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE - 1);
+            upper[i] = randomIntBetween(lower[i], Integer.MAX_VALUE);
+        }
+        return IntPoint.newRangeQuery(field, lower, upper);
+    }
+
+    private Query randomLongPointRangequery(String field, int dims) {
+        long[] lower = new long[dims];
+        long[] upper = new long[dims];
+        for (int i = 0; i < dims; i++) {
+            lower[i] = randomLongBetween(Long.MIN_VALUE, Long.MAX_VALUE - 1);
+            upper[i] = randomLongBetween(lower[i], Long.MAX_VALUE);
+        }
+        return LongPoint.newRangeQuery(field, lower, upper);
+    }
+
+    private Query randomDoublePointRangequery(String field, int dims) {
+        double[] lower = new double[dims];
+        double[] upper = new double[dims];
+        for (int i = 0; i < dims; i++) {
+            lower[i] = randomDoubleBetween(Double.MIN_VALUE, 0, true);
+            upper[i] = randomDoubleBetween(lower[i], Double.MAX_VALUE, true);
+        }
+        return DoublePoint.newRangeQuery(field, lower, upper);
+    }
+
+    private Query merge(Query lhs, Query rhs) {
+        assertThat("error in test assumptions", lhs, instanceOf(PointRangeQuery.class));
+        assertThat("error in test assumptions", rhs, instanceOf(PointRangeQuery.class));
+        return MergedPointRangeQuery.merge((PointRangeQuery) lhs, (PointRangeQuery) rhs);
+    }
+
+    private MergedPointRangeQuery mergeToMergedQuery(Query lhs, Query rhs) {
+        Query merged = merge(lhs, rhs);
+        assertThat(merged, instanceOf(MergedPointRangeQuery.class));
+        return (MergedPointRangeQuery) merged;
+    }
+
+    private boolean matches1d(Query query, long... values) throws IOException {
+        try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
+            List<IndexableField> doc = new ArrayList<>();
+            for (long v : values) {
+                doc.add(new LongPoint("a", v));
+            }
+            iw.addDocument(doc);
+            try (IndexReader r = iw.getReader()) {
+                IndexSearcher searcher = new IndexSearcher(r);
+                return searcher.count(query) > 0;
+            }
+        }
+    }
+
+    private boolean matches2d(Query query, long... values) throws IOException {
+        try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
+            List<IndexableField> doc = new ArrayList<>();
+            assertEquals(values.length % 2, 0);
+            for (int i = 0; i < values.length; i += 2) {
+                doc.add(new LongPoint("a", values[i], values[i + 1]));
+            }
+            iw.addDocument(doc);
+            try (IndexReader r = iw.getReader()) {
+                IndexSearcher searcher = new IndexSearcher(r);
+                return searcher.count(query) > 0;
+            }
+        }
+    }
+
+    private void assertDelegateForSingleValuedSegmentsEqualPointRange(MergedPointRangeQuery actual, Query expected) {
+        /*
+         * This is a lot like asserThat(actual.delegateForSingleValuedSegments(), equalTo(expected)); but
+         * that doesn't work because the subclasses aren't the same. 
+         */
+        assertThat(expected, instanceOf(PointRangeQuery.class));
+        assertThat(actual.delegateForSingleValuedSegments(), instanceOf(PointRangeQuery.class));
+        assertThat(
+            ((PointRangeQuery) actual.delegateForSingleValuedSegments()).getLowerPoint(),
+            equalTo(((PointRangeQuery) expected).getLowerPoint())
+        );
+        assertThat(
+            ((PointRangeQuery) actual.delegateForSingleValuedSegments()).getUpperPoint(),
+            equalTo(((PointRangeQuery) expected).getUpperPoint())
+        );
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java
index e0d644c9fdf18..6f01e4d85c5c0 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java
@@ -25,6 +25,7 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -97,9 +98,23 @@ protected final <R extends InternalAggregation> void asSubAggTestCase(
     }
 
     protected final DateFieldMapper.DateFieldType aggregableDateFieldType(boolean useNanosecondResolution, boolean isSearchable) {
-        return new DateFieldMapper.DateFieldType(AGGREGABLE_DATE, isSearchable, false, true,
-            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
+        return aggregableDateFieldType(useNanosecondResolution, isSearchable, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER);
+    }
+
+    protected final DateFieldMapper.DateFieldType aggregableDateFieldType(
+        boolean useNanosecondResolution,
+        boolean isSearchable,
+        DateFormatter formatter
+    ) {
+        return new DateFieldMapper.DateFieldType(
+            AGGREGABLE_DATE,
+            isSearchable,
+            randomBoolean(),
+            true,
+            formatter,
             useNanosecondResolution ? DateFieldMapper.Resolution.NANOSECONDS : DateFieldMapper.Resolution.MILLISECONDS,
-            null, Collections.emptyMap());
+            null,
+            Collections.emptyMap()
+        );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
index d297d9fabc613..5bc00a2e7dc28 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
@@ -30,13 +30,17 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
+import org.elasticsearch.search.internal.SearchContext;
+import org.hamcrest.Matcher;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -44,9 +48,12 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.function.Consumer;
+import java.util.stream.IntStream;
 
 import static java.util.stream.Collectors.toList;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
 
 public class DateHistogramAggregatorTests extends DateHistogramAggregatorTestCase {
     /**
@@ -1140,6 +1147,84 @@ public void testOverlappingBounds() {
             "hard bounds: [2010-01-01--2020-01-01], extended bounds: [2009-01-01--2021-01-01]"));
     }
 
+    public void testFewRoundingPointsUsesFromRange() throws IOException {
+        aggregationImplementationChoiceTestCase(
+            aggregableDateFieldType(false, true, DateFormatter.forPattern("yyyy")),
+            IntStream.range(2000, 2010).mapToObj(Integer::toString).collect(toList()),
+            new DateHistogramAggregationBuilder("test").field(AGGREGABLE_DATE).calendarInterval(DateHistogramInterval.YEAR),
+            true
+        );
+    }
+
+    public void testManyRoundingPointsDoesNotUseFromRange() throws IOException {
+        aggregationImplementationChoiceTestCase(
+            aggregableDateFieldType(false, true, DateFormatter.forPattern("yyyy")),
+            IntStream.range(2000, 3000).mapToObj(Integer::toString).collect(toList()),
+            new DateHistogramAggregationBuilder("test").field(AGGREGABLE_DATE).calendarInterval(DateHistogramInterval.YEAR),
+            false
+        );
+    }
+
+    /**
+     * Nanos doesn't use from range, but we don't get the fancy compile into
+     * filters because of potential loss of precision.
+     */
+    public void testNanosDoesUseFromRange() throws IOException {
+        aggregationImplementationChoiceTestCase(
+            aggregableDateFieldType(true, true, DateFormatter.forPattern("yyyy")),
+            List.of("2017", "2018"),
+            new DateHistogramAggregationBuilder("test").field(AGGREGABLE_DATE).calendarInterval(DateHistogramInterval.YEAR),
+            true
+        );
+    }
+
+    public void testFarFutureDoesNotUseFromRange() throws IOException {
+        aggregationImplementationChoiceTestCase(
+            aggregableDateFieldType(false, true, DateFormatter.forPattern("yyyyyy")),
+            List.of("402017", "402018"),
+            new DateHistogramAggregationBuilder("test").field(AGGREGABLE_DATE).calendarInterval(DateHistogramInterval.YEAR),
+            false
+        );
+    }
+
+    public void testMissingValueDoesNotUseFromRange() throws IOException {
+        aggregationImplementationChoiceTestCase(
+            aggregableDateFieldType(false, true, DateFormatter.forPattern("yyyy")),
+            List.of("2017", "2018"),
+            new DateHistogramAggregationBuilder("test").field(AGGREGABLE_DATE).calendarInterval(DateHistogramInterval.YEAR).missing("2020"),
+            false
+        );
+    }
+
+    private void aggregationImplementationChoiceTestCase(
+        DateFieldMapper.DateFieldType ft,
+        List<String> data,
+        DateHistogramAggregationBuilder builder,
+        boolean usesFromRange
+    ) throws IOException {
+        try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+            for (String d : data) {
+                long instant = asLong(d, ft);
+                indexWriter.addDocument(
+                    List.of(new SortedNumericDocValuesField(AGGREGABLE_DATE, instant), new LongPoint(AGGREGABLE_DATE, instant))
+                );
+            }
+            try (IndexReader reader = indexWriter.getReader()) {
+                SearchContext context = createSearchContext(new IndexSearcher(reader), new MatchAllDocsQuery(), ft);
+                Aggregator agg = createAggregator(builder, context);
+                Matcher<Aggregator> matcher = instanceOf(DateHistogramAggregator.FromDateRange.class);
+                if (usesFromRange == false) {
+                    matcher = not(matcher);
+                }
+                assertThat(agg, matcher);
+                agg.preCollection();
+                context.searcher().search(context.query(), agg);
+                InternalDateHistogram result = (InternalDateHistogram) agg.buildTopLevel();
+                assertThat(result.getBuckets().stream().map(InternalDateHistogram.Bucket::getKeyAsString).collect(toList()), equalTo(data));
+            }
+        }
+    }
+
     public void testIllegalInterval() throws IOException {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testSearchCase(new MatchAllDocsQuery(),
             Collections.emptyList(),
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java
index 00def8a016ef8..00ed8e34c7b0f 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.search.aggregations.bucket.range;
 
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
@@ -37,6 +39,7 @@
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.DateFieldMapper.Resolution;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
@@ -51,14 +54,16 @@
 import java.util.function.Consumer;
 
 import static java.util.Collections.singleton;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 
 public class DateRangeAggregatorTests extends AggregatorTestCase {
 
     private static final String NUMBER_FIELD_NAME = "number";
     private static final String DATE_FIELD_NAME = "date";
 
-    private Instant t1 = ZonedDateTime.of(2015, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant();
-    private Instant t2 = ZonedDateTime.of(2016, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant();
+    private static final Instant T1 = ZonedDateTime.of(2015, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant();
+    private static final Instant T2 = ZonedDateTime.of(2016, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant();
 
     public void testNoMatchingField() throws IOException {
         testBothResolutions(new MatchAllDocsQuery(), (iw, resolution) -> {
@@ -76,8 +81,18 @@ public void testNoMatchingField() throws IOException {
 
     public void testMatchesSortedNumericDocValues() throws IOException {
         testBothResolutions(new MatchAllDocsQuery(), (iw, resolution) -> {
-            iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, resolution.convert(t1))));
-            iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, resolution.convert(t2))));
+            iw.addDocument(
+                List.of(
+                    new SortedNumericDocValuesField(DATE_FIELD_NAME, resolution.convert(T1)),
+                    new LongPoint(DATE_FIELD_NAME, resolution.convert(T1))
+                )
+            );
+            iw.addDocument(
+                List.of(
+                    new SortedNumericDocValuesField(DATE_FIELD_NAME, resolution.convert(T2)),
+                    new LongPoint(DATE_FIELD_NAME, resolution.convert(T2))
+                )
+            );
         }, range -> {
             List<? extends InternalRange.Bucket> ranges = range.getBuckets();
             assertEquals(2, ranges.size());
@@ -89,8 +104,18 @@ public void testMatchesSortedNumericDocValues() throws IOException {
 
     public void testMatchesNumericDocValues() throws IOException {
         testBothResolutions(new MatchAllDocsQuery(), (iw, resolution) -> {
-            iw.addDocument(singleton(new NumericDocValuesField(DATE_FIELD_NAME, resolution.convert(t1))));
-            iw.addDocument(singleton(new NumericDocValuesField(DATE_FIELD_NAME, resolution.convert(t2))));
+            iw.addDocument(
+                List.of(
+                    new NumericDocValuesField(DATE_FIELD_NAME, resolution.convert(T1)),
+                    new LongPoint(DATE_FIELD_NAME, resolution.convert(T1))
+                )
+            );
+            iw.addDocument(
+                List.of(
+                    new NumericDocValuesField(DATE_FIELD_NAME, resolution.convert(T2)),
+                    new LongPoint(DATE_FIELD_NAME, resolution.convert(T2))
+                )
+            );
         }, range -> {
             List<? extends InternalRange.Bucket> ranges = range.getBuckets();
             assertEquals(2, ranges.size());
@@ -109,8 +134,8 @@ public void  testMissingDateStringWithDateField() throws IOException {
             .addRange("2015-11-13", "2015-11-14");
 
         testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
-            iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, t1.toEpochMilli())));
-            iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, t2.toEpochMilli())));
+            iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, T1.toEpochMilli())));
+            iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, T2.toEpochMilli())));
             // Missing will apply to this document
             iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 7)));
         }, range -> {
@@ -121,6 +146,46 @@ public void  testMissingDateStringWithDateField() throws IOException {
         }, fieldType);
     }
 
+    public void testUnboundedRanges() throws IOException {
+        testCase(
+            new RangeAggregationBuilder("name").field(DATE_FIELD_NAME).addUnboundedTo(5).addUnboundedFrom(5),
+            new MatchAllDocsQuery(),
+            iw -> {
+                iw.addDocument(
+                    List.of(new NumericDocValuesField(DATE_FIELD_NAME, Long.MIN_VALUE), new LongPoint(DATE_FIELD_NAME, Long.MIN_VALUE))
+                );
+                iw.addDocument(List.of(new NumericDocValuesField(DATE_FIELD_NAME, 7), new LongPoint(DATE_FIELD_NAME, 7)));
+                iw.addDocument(List.of(new NumericDocValuesField(DATE_FIELD_NAME, 2), new LongPoint(DATE_FIELD_NAME, 2)));
+                iw.addDocument(List.of(new NumericDocValuesField(DATE_FIELD_NAME, 3), new LongPoint(DATE_FIELD_NAME, 3)));
+                iw.addDocument(
+                    List.of(new NumericDocValuesField(DATE_FIELD_NAME, Long.MAX_VALUE), new LongPoint(DATE_FIELD_NAME, Long.MAX_VALUE))
+                );
+            },
+            result -> {
+                InternalRange<?, ?> range = (InternalRange<?, ?>) result;
+                List<? extends InternalRange.Bucket> ranges = range.getBuckets();
+                assertThat(ranges, hasSize(2));
+                assertThat(ranges.get(0).getFrom(), equalTo(Double.NEGATIVE_INFINITY));
+                assertThat(ranges.get(0).getTo(), equalTo(5d));
+                assertThat(ranges.get(0).getDocCount(), equalTo(3L));
+                assertThat(ranges.get(1).getFrom(), equalTo(5d));
+                assertThat(ranges.get(1).getTo(), equalTo(Double.POSITIVE_INFINITY));
+                assertThat(ranges.get(1).getDocCount(), equalTo(2L));
+                assertTrue(AggregationInspectionHelper.hasValue(range));
+            },
+            new DateFieldMapper.DateFieldType(
+                DATE_FIELD_NAME,
+                randomBoolean(),
+                randomBoolean(),
+                true,
+                DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
+                Resolution.MILLISECONDS,
+                null,
+                null
+            )
+        );
+    }
+
     public void  testNumberFieldDateRanges() throws IOException {
         DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("date_range")
             .field(NUMBER_FIELD_NAME)
@@ -145,8 +210,8 @@ public void  testNumberFieldNumberRanges() throws IOException {
             = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER);
 
         testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
-            iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
-            iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 1)));
+            iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 7), new IntPoint(NUMBER_FIELD_NAME, 7)));
+            iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 1), new IntPoint(NUMBER_FIELD_NAME, 1)));
         }, range -> {
             List<? extends InternalRange.Bucket> ranges = range.getBuckets();
             assertEquals(1, ranges.size());
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java
index dda30646a6ca5..7e00046ad1b61 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.search.aggregations.bucket.range;
 
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
@@ -32,9 +34,11 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.DateFieldMapper.Resolution;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.CardinalityUpperBound;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
@@ -48,6 +52,7 @@
 
 import static java.util.Collections.singleton;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 
 public class RangeAggregatorTests extends AggregatorTestCase {
 
@@ -70,9 +75,9 @@ public void testNoMatchingField() throws IOException {
 
     public void testMatchesSortedNumericDocValues() throws IOException {
         testCase(new MatchAllDocsQuery(), iw -> {
-            iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 7)));
-            iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 2)));
-            iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 3)));
+            iw.addDocument(List.of(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 7), new IntPoint(NUMBER_FIELD_NAME, 7)));
+            iw.addDocument(List.of(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 2), new IntPoint(NUMBER_FIELD_NAME, 2)));
+            iw.addDocument(List.of(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, 3), new IntPoint(NUMBER_FIELD_NAME, 3)));
         }, range -> {
             List<? extends InternalRange.Bucket> ranges = range.getBuckets();
             assertEquals(2, ranges.size());
@@ -84,9 +89,9 @@ public void testMatchesSortedNumericDocValues() throws IOException {
 
     public void testMatchesNumericDocValues() throws IOException {
         testCase(new MatchAllDocsQuery(), iw -> {
-            iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
-            iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 2)));
-            iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 3)));
+            iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 7), new IntPoint(NUMBER_FIELD_NAME, 7)));
+            iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 2), new IntPoint(NUMBER_FIELD_NAME, 2)));
+            iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 3), new IntPoint(NUMBER_FIELD_NAME, 3)));
         }, range -> {
             List<? extends InternalRange.Bucket> ranges = range.getBuckets();
             assertEquals(2, ranges.size());
@@ -96,8 +101,63 @@ public void testMatchesNumericDocValues() throws IOException {
         });
     }
 
+    public void testUnboundedRanges() throws IOException {
+        testCase(
+            new RangeAggregationBuilder("name").field(NUMBER_FIELD_NAME).addUnboundedTo(5).addUnboundedFrom(5),
+            new MatchAllDocsQuery(),
+            iw -> {
+                iw.addDocument(
+                    List.of(
+                        new NumericDocValuesField(NUMBER_FIELD_NAME, Integer.MIN_VALUE),
+                        new IntPoint(NUMBER_FIELD_NAME, Integer.MIN_VALUE)
+                    )
+                );
+                iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 7), new IntPoint(NUMBER_FIELD_NAME, 7)));
+                iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 2), new IntPoint(NUMBER_FIELD_NAME, 2)));
+                iw.addDocument(List.of(new NumericDocValuesField(NUMBER_FIELD_NAME, 3), new IntPoint(NUMBER_FIELD_NAME, 3)));
+                iw.addDocument(
+                    List.of(
+                        new NumericDocValuesField(NUMBER_FIELD_NAME, Integer.MAX_VALUE),
+                        new IntPoint(NUMBER_FIELD_NAME, Integer.MAX_VALUE)
+                    )
+                );
+            },
+            result -> {
+                InternalRange<?, ?> range = (InternalRange<?, ?>) result;
+                List<? extends InternalRange.Bucket> ranges = range.getBuckets();
+                assertThat(ranges, hasSize(2));
+                assertThat(ranges.get(0).getFrom(), equalTo(Double.NEGATIVE_INFINITY));
+                assertThat(ranges.get(0).getTo(), equalTo(5d));
+                assertThat(ranges.get(0).getDocCount(), equalTo(3L));
+                assertThat(ranges.get(1).getFrom(), equalTo(5d));
+                assertThat(ranges.get(1).getTo(), equalTo(Double.POSITIVE_INFINITY));
+                assertThat(ranges.get(1).getDocCount(), equalTo(2L));
+                assertTrue(AggregationInspectionHelper.hasValue(range));
+            },
+            new NumberFieldMapper.NumberFieldType(
+                NUMBER_FIELD_NAME,
+                NumberFieldMapper.NumberType.INTEGER,
+                randomBoolean(),
+                randomBoolean(),
+                true,
+                false,
+                null,
+                null
+            )
+        );
+    }
+
     public void testDateFieldMillisecondResolution() throws IOException {
-        DateFieldMapper.DateFieldType fieldType = new DateFieldMapper.DateFieldType(DATE_FIELD_NAME);
+        DateFieldMapper.DateFieldType fieldType = new DateFieldMapper.DateFieldType(
+            DATE_FIELD_NAME,
+            randomBoolean(),
+            randomBoolean(),
+            true,
+            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
+            Resolution.MILLISECONDS,
+            null,
+            null
+        );
 
         long milli1 = ZonedDateTime.of(2015, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
         long milli2 = ZonedDateTime.of(2016, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
@@ -107,8 +167,8 @@ public void testDateFieldMillisecondResolution() throws IOException {
             .addRange(milli1 - 1, milli1 + 1);
 
         testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
-            iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli1)));
-            iw.addDocument(singleton(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli2)));
+            iw.addDocument(List.of(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli1), new LongPoint(DATE_FIELD_NAME, milli1)));
+            iw.addDocument(List.of(new SortedNumericDocValuesField(DATE_FIELD_NAME, milli2), new LongPoint(DATE_FIELD_NAME, milli2)));
         }, range -> {
             List<? extends InternalRange.Bucket> ranges = range.getBuckets();
             assertEquals(1, ranges.size());
@@ -168,6 +228,39 @@ public void  testMissingDateWithDateField() throws IOException {
         }, fieldType);
     }
 
+    public void testNotFitIntoDouble() throws IOException {
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
+            NUMBER_FIELD_NAME,
+            NumberType.LONG,
+            true,
+            false,
+            true,
+            false,
+            null,
+            null
+        );
+
+        long start = 2L << 54; // Double stores 53 bits of mantissa, so we aggregate a bunch of bigger values
+
+        RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("range")
+            .field(NUMBER_FIELD_NAME)
+            .addRange(start, start + 50)
+            .addRange(start + 50, start + 100)
+            .addUnboundedFrom(start + 100);
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            for (long l = start; l < start + 150; l++) {
+                iw.addDocument(List.of(new SortedNumericDocValuesField(NUMBER_FIELD_NAME, l), new LongPoint(NUMBER_FIELD_NAME, l)));
+            }
+        }, range -> {
+            List<? extends InternalRange.Bucket> ranges = range.getBuckets();
+            assertThat(ranges, hasSize(3));
+            // If we had a native `double` range aggregator we'd get 50, 50, 50
+            assertThat(ranges.stream().mapToLong(InternalRange.Bucket::getDocCount).toArray(), equalTo(new long[] {44, 48, 58}));
+            assertTrue(AggregationInspectionHelper.hasValue(range));
+        }, fieldType);
+    }
+
     public void  testMissingDateWithNumberField() throws IOException {
         RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("range")
             .field(NUMBER_FIELD_NAME)
@@ -295,11 +388,48 @@ public void testSubAggCollectsFromManyBucketsIfManyRanges() throws IOException {
         });
     }
 
+    public void testOverlappingRanges() throws IOException {
+        RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("test_range_agg");
+        aggregationBuilder.field(NUMBER_FIELD_NAME);
+        aggregationBuilder.addRange(0d, 5d);
+        aggregationBuilder.addRange(10d, 20d);
+        aggregationBuilder.addRange(0d, 20d);
+        aggregationBuilder.missing(100);            // Set a missing value to force the "normal" range collection instead of filter-based
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 11)));
+            iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 7)));
+            iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 2)));
+            iw.addDocument(singleton(new NumericDocValuesField(NUMBER_FIELD_NAME, 3)));
+        }, result -> {
+            InternalRange<?, ?> range = (InternalRange<?, ?>) result;
+            List<? extends InternalRange.Bucket> ranges = range.getBuckets();
+            assertThat(ranges, hasSize(3));
+            assertThat(ranges.get(0).getFrom(), equalTo(0d));
+            assertThat(ranges.get(0).getTo(), equalTo(5d));
+            assertThat(ranges.get(0).getDocCount(), equalTo(2L));
+            assertThat(ranges.get(1).getFrom(), equalTo(00d));
+            assertThat(ranges.get(1).getTo(), equalTo(20d));
+            assertThat(ranges.get(1).getDocCount(), equalTo(4L));
+            assertThat(ranges.get(2).getFrom(), equalTo(10d));
+            assertThat(ranges.get(2).getTo(), equalTo(20d));
+            assertThat(ranges.get(2).getDocCount(), equalTo(1L));
+            assertTrue(AggregationInspectionHelper.hasValue(range));
+        }, new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER));
+    }
+
     private void testCase(Query query,
                           CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
                           Consumer<InternalRange<? extends InternalRange.Bucket, ? extends InternalRange>> verify) throws IOException {
-        MappedFieldType fieldType
-            = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER);
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
+            NUMBER_FIELD_NAME,
+            NumberFieldMapper.NumberType.INTEGER,
+            randomBoolean(),
+            randomBoolean(),
+            true,
+            false,
+            null,
+            null
+        );
         RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("test_range_agg");
         aggregationBuilder.field(NUMBER_FIELD_NAME);
         aggregationBuilder.addRange(0d, 5d);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java
index 9d2f38347b40e..36bff44170b4c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java
@@ -73,7 +73,7 @@ public void testDatePrepareRoundingWithQuery() throws IOException {
         MapperService mapperService = dateMapperService();
         Query query = mapperService.fieldType("field")
             .rangeQuery(min, max, true, true, ShapeRelation.CONTAINS, null, null, createQueryShardContext(mapperService));
-        withAggregationContext(mapperService, List.of(), query, context -> {
+        withAggregationContext(null, mapperService, List.of(), query, context -> {
             Rounding rounding = mock(Rounding.class);
             CoreValuesSourceType.DATE.getField(context.buildFieldContext("field"), null, context).roundingPreparer().apply(rounding);
             verify(rounding).prepare(min, max);
@@ -102,7 +102,7 @@ public void testDatePrepareRoundingWithDocAndQuery() throws IOException {
         MapperService mapperService = dateMapperService();
         Query query = mapperService.fieldType("field")
             .rangeQuery(minQuery, maxQuery, true, true, ShapeRelation.CONTAINS, null, null, createQueryShardContext(mapperService));
-        withAggregationContext(mapperService, docsWithDatesBetween(minDocs, maxDocs), query, context -> {
+        withAggregationContext(null, mapperService, docsWithDatesBetween(minDocs, maxDocs), query, context -> {
             Rounding rounding = mock(Rounding.class);
             CoreValuesSourceType.DATE.getField(context.buildFieldContext("field"), null, context).roundingPreparer().apply(rounding);
             verify(rounding).prepare(min, max);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java
index 3ac362764ecec..d1caaacafe077 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java
@@ -55,12 +55,15 @@ public void testEmptyKeyword() throws Exception {
             LeafReaderContext ctx = context.searcher().getIndexReader().leaves().get(0);
             SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
             assertFalse(values.advanceExact(0));
+            assertTrue(config.alignesWithSearchIndex());
+
             config = ValuesSourceConfig.resolve(context, null, "field", null, "abc", null, null, CoreValuesSourceType.BYTES);
             valuesSource = (ValuesSource.Bytes) config.getValuesSource();
             values = valuesSource.bytesValues(ctx);
             assertTrue(values.advanceExact(0));
             assertEquals(1, values.docValueCount());
             assertEquals(new BytesRef("abc"), values.nextValue());
+            assertFalse(config.alignesWithSearchIndex());
         });
     }
 
@@ -72,6 +75,7 @@ public void testUnmappedKeyword() throws Exception {
             ValuesSource.Bytes valuesSource = (ValuesSource.Bytes) config.getValuesSource();
             assertNotNull(valuesSource);
             assertFalse(config.hasValues());
+            assertFalse(config.alignesWithSearchIndex());
 
             config = ValuesSourceConfig.resolve(context, ValueType.STRING, "field", null, "abc", null, null, CoreValuesSourceType.BYTES);
             valuesSource = (ValuesSource.Bytes) config.getValuesSource();
@@ -80,6 +84,7 @@ public void testUnmappedKeyword() throws Exception {
             assertTrue(values.advanceExact(0));
             assertEquals(1, values.docValueCount());
             assertEquals(new BytesRef("abc"), values.nextValue());
+            assertFalse(config.alignesWithSearchIndex());
         });
     }
 
@@ -94,6 +99,7 @@ public void testLong() throws Exception {
             assertTrue(values.advanceExact(0));
             assertEquals(1, values.docValueCount());
             assertEquals(42, values.nextValue());
+            assertTrue(config.alignesWithSearchIndex());
         });
     }
 
@@ -106,6 +112,7 @@ public void testEmptyLong() throws Exception {
             LeafReaderContext ctx = context.searcher().getIndexReader().leaves().get(0);
             SortedNumericDocValues values = valuesSource.longValues(ctx);
             assertFalse(values.advanceExact(0));
+            assertTrue(config.alignesWithSearchIndex());
 
             config = ValuesSourceConfig.resolve(context, null, "field", null, 42, null, null, CoreValuesSourceType.BYTES);
             valuesSource = (ValuesSource.Numeric) config.getValuesSource();
@@ -113,6 +120,7 @@ public void testEmptyLong() throws Exception {
             assertTrue(values.advanceExact(0));
             assertEquals(1, values.docValueCount());
             assertEquals(42, values.nextValue());
+            assertFalse(config.alignesWithSearchIndex());
         });
     }
 
@@ -124,6 +132,7 @@ public void testUnmappedLong() throws Exception {
             ValuesSource.Numeric valuesSource = (ValuesSource.Numeric) config.getValuesSource();
             assertNotNull(valuesSource);
             assertFalse(config.hasValues());
+            assertFalse(config.alignesWithSearchIndex());
 
             config = ValuesSourceConfig.resolve(context, ValueType.NUMBER, "field", null, 42, null, null, CoreValuesSourceType.BYTES);
             valuesSource = (ValuesSource.Numeric) config.getValuesSource();
@@ -132,6 +141,7 @@ public void testUnmappedLong() throws Exception {
             assertTrue(values.advanceExact(0));
             assertEquals(1, values.docValueCount());
             assertEquals(42, values.nextValue());
+            assertFalse(config.alignesWithSearchIndex());
         });
     }
 
@@ -146,6 +156,7 @@ public void testBoolean() throws Exception {
             assertTrue(values.advanceExact(0));
             assertEquals(1, values.docValueCount());
             assertEquals(1, values.nextValue());
+            assertTrue(config.alignesWithSearchIndex());
         });
     }
 
@@ -158,6 +169,7 @@ public void testEmptyBoolean() throws Exception {
             LeafReaderContext ctx = context.searcher().getIndexReader().leaves().get(0);
             SortedNumericDocValues values = valuesSource.longValues(ctx);
             assertFalse(values.advanceExact(0));
+            assertTrue(config.alignesWithSearchIndex());
 
             config = ValuesSourceConfig.resolve(context, null, "field", null, true, null, null, CoreValuesSourceType.BYTES);
             valuesSource = (ValuesSource.Numeric) config.getValuesSource();
@@ -165,6 +177,7 @@ public void testEmptyBoolean() throws Exception {
             assertTrue(values.advanceExact(0));
             assertEquals(1, values.docValueCount());
             assertEquals(1, values.nextValue());
+            assertFalse(config.alignesWithSearchIndex());
         });
     }
 
@@ -176,6 +189,7 @@ public void testUnmappedBoolean() throws Exception {
             ValuesSource.Numeric valuesSource = (ValuesSource.Numeric) config.getValuesSource();
             assertNotNull(valuesSource);
             assertFalse(config.hasValues());
+            assertFalse(config.alignesWithSearchIndex());
 
             config = ValuesSourceConfig.resolve(context, ValueType.BOOLEAN, "field", null, true, null, null, CoreValuesSourceType.BYTES);
             valuesSource = (ValuesSource.Numeric) config.getValuesSource();
@@ -184,6 +198,7 @@ public void testUnmappedBoolean() throws Exception {
             assertTrue(values.advanceExact(0));
             assertEquals(1, values.docValueCount());
             assertEquals(1, values.nextValue());
+            assertFalse(config.alignesWithSearchIndex());
         });
     }
 
@@ -214,6 +229,7 @@ public void testFieldAlias() throws Exception {
             assertTrue(values.advanceExact(0));
             assertEquals(1, values.docValueCount());
             assertEquals(new BytesRef("value"), values.nextValue());
+            assertTrue(config.alignesWithSearchIndex());
         });
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
index d8f4d15c9770f..d3b236df26173 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
@@ -281,7 +281,12 @@ protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IO
         });
     }
 
-    private AggregationContext aggregationContext(MapperService mapperService, IndexSearcher searcher, Query query) {
+    private AggregationContext aggregationContext(
+        ValuesSourceRegistry valuesSourceRegistry,
+        MapperService mapperService,
+        IndexSearcher searcher,
+        Query query
+    ) {
         return new AggregationContext() {
             @Override
             public IndexSearcher searcher() {
@@ -310,7 +315,7 @@ public SearchLookup lookup() {
 
             @Override
             public ValuesSourceRegistry getValuesSourceRegistry() {
-                throw new UnsupportedOperationException();
+                return valuesSourceRegistry;
             }
 
             @Override
@@ -365,10 +370,11 @@ protected final void withAggregationContext(
         List<SourceToParse> docs,
         CheckedConsumer<AggregationContext, IOException> test
     ) throws IOException {
-        withAggregationContext(mapperService, docs, null, test);
+        withAggregationContext(null, mapperService, docs, null, test);
     }
 
     protected final void withAggregationContext(
+        ValuesSourceRegistry valuesSourceRegistry,
         MapperService mapperService,
         List<SourceToParse> docs,
         Query query,
@@ -381,7 +387,7 @@ protected final void withAggregationContext(
                     writer.addDocuments(mapperService.documentMapper().parse(doc).docs());
                 }
             },
-            reader -> test.accept(aggregationContext(mapperService, new IndexSearcher(reader), query))
+            reader -> test.accept(aggregationContext(valuesSourceRegistry, mapperService, new IndexSearcher(reader), query))
         );
     }
 
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java
index d6d649b34e4bb..fd3ff11b5bef0 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java
@@ -6,23 +6,9 @@
 
 package org.elasticsearch.xpack.analytics.rate;
 
-import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues;
-import static org.hamcrest.Matchers.closeTo;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.instanceOf;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Consumer;
-import java.util.function.Function;
-
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
@@ -59,6 +45,22 @@
 import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
 import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.analytics.AnalyticsTestsUtils.histogramFieldDocValues;
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+
 public class RateAggregatorTests extends AggregatorTestCase {
 
     /**
@@ -331,16 +333,36 @@ public void testKeywordSandwich() throws IOException {
 
         testCase(dateHistogramAggregationBuilder, new MatchAllDocsQuery(), iw -> {
             iw.addDocument(
-                doc("2010-03-11T01:07:45", new NumericDocValuesField("val", 1), new SortedSetDocValuesField("term", new BytesRef("a")))
+                doc(
+                    "2010-03-11T01:07:45",
+                    new NumericDocValuesField("val", 1),
+                    new IntPoint("val", 1),
+                    new SortedSetDocValuesField("term", new BytesRef("a"))
+                )
             );
             iw.addDocument(
-                doc("2010-03-12T01:07:45", new NumericDocValuesField("val", 2), new SortedSetDocValuesField("term", new BytesRef("a")))
+                doc(
+                    "2010-03-12T01:07:45",
+                    new NumericDocValuesField("val", 2),
+                    new IntPoint("val", 2),
+                    new SortedSetDocValuesField("term", new BytesRef("a"))
+                )
             );
             iw.addDocument(
-                doc("2010-04-01T03:43:34", new NumericDocValuesField("val", 3), new SortedSetDocValuesField("term", new BytesRef("a")))
+                doc(
+                    "2010-04-01T03:43:34",
+                    new NumericDocValuesField("val", 3),
+                    new IntPoint("val", 3),
+                    new SortedSetDocValuesField("term", new BytesRef("a"))
+                )
             );
             iw.addDocument(
-                doc("2010-04-27T03:43:34", new NumericDocValuesField("val", 4), new SortedSetDocValuesField("term", new BytesRef("b")))
+                doc(
+                    "2010-04-27T03:43:34",
+                    new NumericDocValuesField("val", 4),
+                    new IntPoint("val", 4),
+                    new SortedSetDocValuesField("term", new BytesRef("b"))
+                )
             );
         }, (Consumer<InternalDateHistogram>) dh -> {
             assertThat(dh.getBuckets(), hasSize(2));
@@ -681,6 +703,7 @@ private Iterable<IndexableField> doc(String date, IndexableField... fields) {
         List<IndexableField> indexableFields = new ArrayList<>();
         long instant = dateFieldType(DATE_FIELD).parse(date);
         indexableFields.add(new SortedNumericDocValuesField(DATE_FIELD, instant));
+        indexableFields.add(new LongPoint(DATE_FIELD, instant));
         indexableFields.addAll(Arrays.asList(fields));
         return indexableFields;
     }
diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java
index 88a5183d9f4c9..f08e6f8104042 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java
+++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongTests.java
@@ -42,6 +42,7 @@
 import static org.elasticsearch.search.aggregations.AggregationBuilders.min;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.hamcrest.Matchers.equalTo;
 

From 2bf3e361440385acbdca9814f2d009940069c08c Mon Sep 17 00:00:00 2001
From: Leaf-Lin <39002973+Leaf-Lin@users.noreply.github.com>
Date: Tue, 10 Nov 2020 06:19:17 +1100
Subject: [PATCH 19/34] remove node.ingest setting in the documentation
 (#64456)

I'm not sure if this setting was left here deliberately? or by accident?
With all other node role definition has changed syntax from `node.xxx` to `node.roles: [ ]`, the ingest one is the only one left behind.
---
 docs/reference/modules/node.asciidoc | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc
index a55b8f23e9c23..d1062e79bb588 100644
--- a/docs/reference/modules/node.asciidoc
+++ b/docs/reference/modules/node.asciidoc
@@ -277,14 +277,6 @@ To create a dedicated ingest node, set:
 node.roles: [ ingest ]
 ----
 
-[[node-ingest-node-setting]]
-// tag::node-ingest-tag[]
-`node.ingest` {ess-icon}::
-Determines whether a node is an ingest node. <<ingest,Ingest nodes>> can apply
-an ingest pipeline to transform and enrich a document before indexing. Default:
-`true`.
-// end::node-ingest-tag[]
-
 [[coordinating-only-node]]
 ==== Coordinating only node
 

From b92c9b7147dbf04d6cf221f381e892c5871b3bdd Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Mon, 9 Nov 2020 11:41:49 -0800
Subject: [PATCH 20/34] Split test runner security permissions (#64748)

The test framework security policy contains permissions for both gradle
and intellij running tests. These currently coexist in the same file,
though only one set of the jars permissions are granted to exist in any
given run. This works because java policy parsing is lenient, so if a
system property referenced in the file does not exist, the entire grant
is silently skipped. This commit splits these permissions into separate
policy files so that we do not rely on leniency, and can (in a followup)
add our own validation to fix java's leniency.
---
 distribution/build.gradle                     |  1 -
 server/build.gradle                           |  3 +--
 .../org/elasticsearch/bootstrap/gradle.policy | 16 ++++++++++++++
 .../elasticsearch/bootstrap/intellij.policy   |  6 +++++
 .../bootstrap/test-framework.policy           | 22 -------------------
 .../bootstrap/BootstrapForTesting.java        | 15 ++++++++++---
 6 files changed, 35 insertions(+), 28 deletions(-)
 create mode 100644 server/src/main/resources/org/elasticsearch/bootstrap/gradle.policy
 create mode 100644 server/src/main/resources/org/elasticsearch/bootstrap/intellij.policy

diff --git a/distribution/build.gradle b/distribution/build.gradle
index 29f6036195b87..5d74ea0de610b 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -335,7 +335,6 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
 
   dependencies {
     libs project(':server')
-    libs project(':libs:elasticsearch-plugin-classloader')
     libs project(':distribution:tools:java-version-checker')
     libs project(':distribution:tools:launchers')
 
diff --git a/server/build.gradle b/server/build.gradle
index fb28a10e0dc84..ddf7e0420309c 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -39,8 +39,7 @@ dependencies {
   api project(':libs:elasticsearch-x-content')
   api project(":libs:elasticsearch-geo")
 
-  compileOnly project(':libs:elasticsearch-plugin-classloader')
-  testRuntimeOnly project(':libs:elasticsearch-plugin-classloader')
+  implementation project(':libs:elasticsearch-plugin-classloader')
 
   // lucene
   api "org.apache.lucene:lucene-core:${versions.lucene}"
diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/gradle.policy b/server/src/main/resources/org/elasticsearch/bootstrap/gradle.policy
new file mode 100644
index 0000000000000..eae776d22fbbf
--- /dev/null
+++ b/server/src/main/resources/org/elasticsearch/bootstrap/gradle.policy
@@ -0,0 +1,16 @@
+grant codeBase "file:${gradle.dist.lib}/-" {
+  // gradle test worker code needs a slew of permissions, we give full access here since gradle isn't a production
+  // dependency and there's no point in exercising the security policy against it
+  permission java.security.AllPermission;
+};
+
+grant codeBase "file:${gradle.worker.jar}" {
+  // gradle test worker code needs a slew of permissions, we give full access here since gradle isn't a production
+  // dependency and there's no point in exercising the security policy against it
+  permission java.security.AllPermission;
+};
+
+grant {
+  // since the gradle test worker jar is on the test classpath, our tests should be able to read it
+  permission java.io.FilePermission "${gradle.worker.jar}", "read";
+};
diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/intellij.policy b/server/src/main/resources/org/elasticsearch/bootstrap/intellij.policy
new file mode 100644
index 0000000000000..42448c51ec68d
--- /dev/null
+++ b/server/src/main/resources/org/elasticsearch/bootstrap/intellij.policy
@@ -0,0 +1,6 @@
+grant codeBase "${codebase.junit-rt.jar}" {
+  // allows IntelliJ IDEA JUnit test runner to control number of test iterations
+  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+};
+
+
diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index b195662dcf20e..4a8647e5a0474 100644
--- a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -88,25 +88,3 @@ grant codeBase "${codebase.httpasyncclient}" {
   // rest client uses system properties which gets the default proxy
   permission java.net.NetPermission "getProxySelector";
 };
-
-grant codeBase "${codebase.junit-rt.jar}" {
-  // allows IntelliJ IDEA JUnit test runner to control number of test iterations
-  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
-};
-
-grant codeBase "file:${gradle.dist.lib}/-" {
-  // gradle test worker code needs a slew of permissions, we give full access here since gradle isn't a production
-  // dependency and there's no point in exercising the security policy against it
-  permission java.security.AllPermission;
-};
-
-grant codeBase "file:${gradle.worker.jar}" {
-  // gradle test worker code needs a slew of permissions, we give full access here since gradle isn't a production
-  // dependency and there's no point in exercising the security policy against it
-  permission java.security.AllPermission;
-};
-
-grant {
-  // since the gradle test worker jar is on the test classpath, our tests should be able to read it
-  permission java.io.FilePermission "${gradle.worker.jar}", "read";
-};
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
index d91b81f4b8ee9..67447fe029862 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
@@ -138,18 +138,26 @@ public class BootstrapForTesting {
                 // TODO: cut over all tests to bind to ephemeral ports
                 perms.add(new SocketPermission("localhost:1024-", "listen,resolve"));
 
+                boolean inGradle = System.getProperty("tests.gradle") != null;
+
                 // read test-framework permissions
                 Map<String, URL> codebases = PolicyUtil.getCodebaseJarMap(JarHell.parseClassPath());
                 // when testing server, the main elasticsearch code is not yet in a jar, so we need to manually add it
                 addClassCodebase(codebases,"elasticsearch", "org.elasticsearch.plugins.PluginsService");
-                if (System.getProperty("tests.gradle") == null) {
+                if (inGradle == false) {
                     // intellij and eclipse don't package our internal libs, so we need to set the codebases for them manually
-                    addClassCodebase(codebases,"plugin-classloader", "org.elasticsearch.plugins.ExtendedPluginsClassLoader");
+                    addClassCodebase(codebases,"elasticsearch-plugin-classloader", "org.elasticsearch.plugins.ExtendedPluginsClassLoader");
                     addClassCodebase(codebases,"elasticsearch-nio", "org.elasticsearch.nio.ChannelFactory");
                     addClassCodebase(codebases, "elasticsearch-secure-sm", "org.elasticsearch.secure_sm.SecureSM");
                     addClassCodebase(codebases, "elasticsearch-rest-client", "org.elasticsearch.client.RestClient");
                 }
                 final Policy testFramework = PolicyUtil.readPolicy(Bootstrap.class.getResource("test-framework.policy"), codebases);
+                final Policy runnerPolicy;
+                if (inGradle) {
+                    runnerPolicy = PolicyUtil.readPolicy(Bootstrap.class.getResource("gradle.policy"), codebases);
+                } else {
+                    runnerPolicy = PolicyUtil.readPolicy(Bootstrap.class.getResource("intellij.policy"), codebases);
+                }
                 // this mimicks the recursive data path permission added in Security.java
                 Permissions fastPathPermissions = new Permissions();
                 addDirectoryPath(fastPathPermissions, "java.io.tmpdir-fastpath", javaTmpDir, "read,readlink,write,delete", true);
@@ -159,7 +167,8 @@ public class BootstrapForTesting {
                     @Override
                     public boolean implies(ProtectionDomain domain, Permission permission) {
                         // implements union
-                        return esPolicy.implies(domain, permission) || testFramework.implies(domain, permission);
+                        return esPolicy.implies(domain, permission) || testFramework.implies(domain, permission) ||
+                            runnerPolicy.implies(domain, permission);
                     }
                 });
                 System.setSecurityManager(SecureSM.createTestSecureSM());

From 82242f7c3f4e8bda8711f1f1b85245e12265a217 Mon Sep 17 00:00:00 2001
From: Dan Hermann <danhermann@users.noreply.github.com>
Date: Mon, 9 Nov 2020 13:43:24 -0600
Subject: [PATCH 21/34] Adjust deprecation version after backport (#64789)

---
 docs/reference/cat/shards.asciidoc                              | 2 +-
 .../src/main/resources/rest-api-spec/api/cat.shards.json        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc
index 46adb53d3222c..7a69cd4da5cce 100644
--- a/docs/reference/cat/shards.asciidoc
+++ b/docs/reference/cat/shards.asciidoc
@@ -276,7 +276,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help]
 `local`::
 (Optional, boolean)
 +
-deprecated::[7.10.0,"This parameter does not affect the request. It will be removed in a future release."]
+deprecated::[7.11.0,"This parameter does not affect the request. It will be removed in a future release."]
 
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
index fcf82e3e8ad82..576cba3fdcd49 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
@@ -53,7 +53,7 @@
         "type":"boolean",
         "description":"Return local information, do not retrieve the state from master node (default: false)",
         "deprecated":{
-          "version":"8.0.0",
+          "version":"7.11.0",
           "description":"This parameter does not affect the request. It will be removed in a future release."
         }
       },

From fae9b06cd5cdb52f714d455156484e6a1cb7b1db Mon Sep 17 00:00:00 2001
From: Dan Hermann <danhermann@users.noreply.github.com>
Date: Mon, 9 Nov 2020 13:43:47 -0600
Subject: [PATCH 22/34] Adjust deprecation version after backport (#64794)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 docs/reference/cat/indices.asciidoc                             | 2 +-
 .../src/main/resources/rest-api-spec/api/cat.indices.json       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc
index f9c9991a60bb1..7c8bdef996b78 100644
--- a/docs/reference/cat/indices.asciidoc
+++ b/docs/reference/cat/indices.asciidoc
@@ -77,7 +77,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segme
 `local`::
 (Optional, boolean)
 +
-deprecated::[7.10.0,"This parameter does not affect the request. It will be removed in a future release."]
+deprecated::[7.11.0,"This parameter does not affect the request. It will be removed in a future release."]
 
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
index 165bf1a3f235a..ed821ca52ed53 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
@@ -53,7 +53,7 @@
         "type":"boolean",
         "description":"Return local information, do not retrieve the state from master node (default: false)",
         "deprecated":{
-          "version":"8.0.0",
+          "version":"7.11.0",
           "description":"This parameter does not affect the request. It will be removed in a future release."
         }
       },

From a573f42f87a218f8fabac26f04eea9e472628ddb Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 9 Nov 2020 15:13:52 -0500
Subject: [PATCH 23/34] Give test a little more space

In #64744 I added a randomized test for how much room a part of the
cardinality use in memory. It's bound were a little low. This gives it a
few more bytes before it falls over so we don't see the test fail 1% or
2% of the time.
---
 .../aggregations/metrics/HyperLogLogPlusPlusSparseTests.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java
index 9faa6cf33c40a..9096e763e6038 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java
@@ -127,7 +127,7 @@ public void testAllocation() {
         int precision = between(MIN_PRECISION, MAX_PRECISION);
         long initialBucketCount = between(0, 100);
         MockBigArrays.assertFitsIn(
-            ByteSizeValue.ofBytes(initialBucketCount * 16),
+            ByteSizeValue.ofBytes(Math.max(256, initialBucketCount * 32)),
             bigArrays -> new HyperLogLogPlusPlusSparse(precision, bigArrays, initialBucketCount)
         );
     }

From cb26c542cee5619d2f2bd86568e12ac76785cdf5 Mon Sep 17 00:00:00 2001
From: James Rodewig <40268737+jrodewig@users.noreply.github.com>
Date: Mon, 9 Nov 2020 16:11:13 -0500
Subject: [PATCH 24/34] [DOCS] Document get pipeline API as multi-target
 (#64816)

---
 docs/reference/ingest/apis/get-pipeline.asciidoc | 8 ++++++--
 docs/reference/rest-api/common-parms.asciidoc    | 6 ------
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/docs/reference/ingest/apis/get-pipeline.asciidoc b/docs/reference/ingest/apis/get-pipeline.asciidoc
index 8039d9648b81b..77b5cf9b9063a 100644
--- a/docs/reference/ingest/apis/get-pipeline.asciidoc
+++ b/docs/reference/ingest/apis/get-pipeline.asciidoc
@@ -45,8 +45,12 @@ GET /_ingest/pipeline/my-pipeline-id
 [[get-pipeline-api-path-params]]
 ==== {api-path-parms-title}
 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=path-pipeline]
-
+`<pipeline>`::
+(Optional, string)
+Comma-separated list of pipeline IDs to retrieve. Wildcard (`*`) expressions are
+supported.
++
+To get all ingest pipelines, omit this parameter or use `*`.
 
 
 [[get-pipeline-api-query-params]]
diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc
index ae7209becfab9..677c3be00dab6 100644
--- a/docs/reference/rest-api/common-parms.asciidoc
+++ b/docs/reference/rest-api/common-parms.asciidoc
@@ -642,12 +642,6 @@ The number of search or bulk index operations processed. Documents are
 processed in batches instead of individually.
 end::pages-processed[]
 
-tag::path-pipeline[]
-`<pipeline>`::
-(Optional, string) Comma-separated list or wildcard expression of pipeline IDs
-used to limit the request.
-end::path-pipeline[]
-
 tag::pivot[]
 The method for transforming the data. These objects define the pivot function
 `group by` fields and the aggregation to reduce the data.

From ca5de06b0c6c75465c366b017ac0d7b56820c0c1 Mon Sep 17 00:00:00 2001
From: Mayya Sharipova <mayya.sharipova@elastic.co>
Date: Mon, 9 Nov 2020 16:27:06 -0500
Subject: [PATCH 25/34] Correct format in analysis-predicate-context

---
 .../painless-analysis-predicate-context.asciidoc              | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc b/docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc
index 3edb1080611d2..55d3818a3462a 100644
--- a/docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc
+++ b/docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc
@@ -30,7 +30,7 @@ analysis chain matches a predicate.
 `token.type` (`String`, read-only)::
         The type of the current token
 
-`token.keyword` ('boolean`, read-only)::
+`token.keyword` (`boolean`, read-only)::
         Whether or not the current token is marked as a keyword
 
 *Return*
@@ -40,4 +40,4 @@ analysis chain matches a predicate.
 
 *API*
 
-The standard <<painless-api-reference-shared, Painless API>> is available.
\ No newline at end of file
+The standard <<painless-api-reference-shared, Painless API>> is available.

From 9dedef081d7c7f6b693a4e6f49fac86d40235205 Mon Sep 17 00:00:00 2001
From: Przemko Robakowski <przemko.robakowski@elastic.co>
Date: Mon, 9 Nov 2020 22:45:43 +0100
Subject: [PATCH 26/34] Use data stream for ILM history (#64521)

This change moves ILM history from using normal index and legacy template to data streams and composable templates.
It also unmutes several ITs to check if using data streams will resolve some race conditions there.
---
 .../test/rest/ESRestTestCase.java             |   3 +-
 .../core/src/main/resources/ilm-history.json  |   8 +-
 .../ilm/TimeSeriesLifecycleActionsIT.java     |   6 +-
 .../xpack/ilm/ILMHistoryTests.java            | 120 ---------
 .../xpack/ilm/history/ILMHistoryStore.java    | 109 +-------
 .../history/ILMHistoryTemplateRegistry.java   |   3 +-
 .../ilm/history/ILMHistoryStoreTests.java     | 233 ++----------------
 7 files changed, 46 insertions(+), 436 deletions(-)
 delete mode 100644 x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMHistoryTests.java

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 0f4afb09e130d..f42c0d2c2881c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -646,7 +646,8 @@ private void wipeCluster() throws Exception {
     protected static void wipeAllIndices() throws IOException {
         boolean includeHidden = minimumNodeVersion().onOrAfter(Version.V_7_7_0);
         try {
-            final Request deleteRequest = new Request("DELETE", "*");
+            //remove all indices except ilm history which can pop up after deleting all data streams but shouldn't interfere
+            final Request deleteRequest = new Request("DELETE", "*,-.ds-ilm-history-*");
             deleteRequest.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : ""));
             RequestOptions allowSystemIndexAccessWarningOptions = RequestOptions.DEFAULT.toBuilder()
                 .setWarningsHandler(warnings -> {
diff --git a/x-pack/plugin/core/src/main/resources/ilm-history.json b/x-pack/plugin/core/src/main/resources/ilm-history.json
index 44ad66d9dabdc..95c3f0133a02f 100644
--- a/x-pack/plugin/core/src/main/resources/ilm-history.json
+++ b/x-pack/plugin/core/src/main/resources/ilm-history.json
@@ -2,15 +2,15 @@
   "index_patterns": [
     "ilm-history-${xpack.ilm_history.template.version}*"
   ],
+  "data_stream": {
+    "hidden": true
+  },
   "template": {
     "settings": {
       "index.number_of_shards": 1,
       "index.number_of_replicas": 0,
       "index.auto_expand_replicas": "0-1",
-      "index.lifecycle.name": "ilm-history-ilm-policy",
-      "index.lifecycle.rollover_alias": "ilm-history-${xpack.ilm_history.template.version}",
-      "index.hidden": true,
-      "index.format": 1
+      "index.lifecycle.name": "ilm-history-ilm-policy"
     },
     "mappings": {
         "dynamic": false,
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
index 3b50caf2fafdb..6895c4e4408cd 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.Nullable;
@@ -1210,7 +1211,6 @@ public void testWaitForActiveShardsStep() throws Exception {
         assertBusy(() -> assertThat(getStepKeyForIndex(client(), originalIndex), equalTo(PhaseCompleteStep.finalStep("hot").getKey())));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/54093")
     public void testHistoryIsWrittenWithSuccess() throws Exception {
         createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, 1L));
         Request createIndexTemplate = new Request("PUT", "_template/rolling_indexes");
@@ -1249,7 +1249,6 @@ public void testHistoryIsWrittenWithSuccess() throws Exception {
         assertBusy(() -> assertHistoryIsPresent(policy, index + "-000002", true, "check-rollover-ready"), 30, TimeUnit.SECONDS);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/50353")
     public void testHistoryIsWrittenWithFailure() throws Exception {
         createIndexWithSettings(client(), index + "-1", alias, Settings.builder(), false);
         createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, 1L));
@@ -1267,7 +1266,6 @@ public void testHistoryIsWrittenWithFailure() throws Exception {
         assertBusy(() -> assertHistoryIsPresent(policy, index + "-1", false, "ERROR"), 30, TimeUnit.SECONDS);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53718")
     public void testHistoryIsWrittenWithDeletion() throws Exception {
         // Index should be created and then deleted by ILM
         createIndexWithSettings(client(), index, alias, Settings.builder(), false);
@@ -1557,7 +1555,7 @@ private void assertHistoryIsPresent(String policyName, String indexName, boolean
         }
 
         // Finally, check that the history index is in a good state
-        Step.StepKey stepKey = getStepKeyForIndex(client(), "ilm-history-2-000001");
+        Step.StepKey stepKey = getStepKeyForIndex(client(), DataStream.getDefaultBackingIndexName("ilm-history-5", 1));
         assertEquals("hot", stepKey.getPhase());
         assertEquals(RolloverAction.NAME, stepKey.getAction());
         assertEquals(WaitForRolloverReadyStep.NAME, stepKey.getName());
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMHistoryTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMHistoryTests.java
deleted file mode 100644
index 808852a9b8411..0000000000000
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMHistoryTests.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.ilm;
-
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
-import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
-import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
-import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
-import org.elasticsearch.xpack.core.ilm.OperationMode;
-import org.elasticsearch.xpack.core.ilm.Phase;
-import org.elasticsearch.xpack.core.ilm.StopILMRequest;
-import org.elasticsearch.xpack.core.ilm.action.GetStatusAction;
-import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
-import org.elasticsearch.xpack.core.ilm.action.StopILMAction;
-import org.elasticsearch.xpack.ilm.history.ILMHistoryStore;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.concurrent.TimeUnit;
-
-import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
-import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.is;
-
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
-public class ILMHistoryTests extends ESIntegTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal));
-        settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
-        settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false);
-        settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
-        settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
-        settings.put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s");
-        settings.put(LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING.getKey(), false);
-        return settings.build();
-    }
-
-    @Override
-    protected boolean ignoreExternalCluster() {
-        return true;
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(LocalStateCompositeXPackPlugin.class, IndexLifecycle.class);
-    }
-
-    private void putTestPolicy() throws InterruptedException, java.util.concurrent.ExecutionException {
-        Phase phase = new Phase("hot", TimeValue.ZERO, Collections.emptyMap());
-        LifecyclePolicy lifecyclePolicy = new LifecyclePolicy("test", Collections.singletonMap("hot", phase));
-        PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy);
-        assertAcked(client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get());
-    }
-
-    public void testIlmHistoryIndexCanRollover() throws Exception {
-        putTestPolicy();
-        Settings settings = Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_SHARDS, 1)
-            .put(SETTING_NUMBER_OF_REPLICAS, 0).put(LifecycleSettings.LIFECYCLE_NAME, "test").build();
-        CreateIndexResponse res = client().admin().indices().prepareCreate("test").setSettings(settings).get();
-        assertTrue(res.isAcknowledged());
-
-        String firstIndex = ILMHistoryStore.ILM_HISTORY_INDEX_PREFIX + "000001";
-        String secondIndex = ILMHistoryStore.ILM_HISTORY_INDEX_PREFIX + "000002";
-
-        assertBusy(() -> {
-            try {
-                GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(firstIndex).get();
-                assertThat(getIndexResponse.getIndices(), arrayContaining(firstIndex));
-            } catch (Exception e) {
-                fail(e.getMessage());
-            }
-        });
-
-        //wait for all history items to index to avoid waiting for timeout in ILMHistoryStore beforeBulk
-        assertBusy(() -> {
-            try {
-                SearchResponse search = client().prepareSearch(firstIndex).setQuery(matchQuery("index", firstIndex)).setSize(0).get();
-                assertHitCount(search, 9);
-            } catch (Exception e) {
-                //assertBusy will stop on first non-assertion error and it can happen when we try to search too early
-                //instead of failing the whole test change it to assertion error and wait some more time
-                fail(e.getMessage());
-            }
-        }, 1L, TimeUnit.MINUTES);
-
-        //make sure ILM is stopped so no new items will be queued in ILM history
-        assertTrue(client().execute(StopILMAction.INSTANCE, new StopILMRequest()).actionGet().isAcknowledged());
-        assertBusy(() -> {
-            GetStatusAction.Response status = client().execute(GetStatusAction.INSTANCE, new GetStatusAction.Request()).actionGet();
-            assertThat(status.getMode(), is(OperationMode.STOPPED));
-        });
-
-        RolloverResponse rolloverResponse = client().admin().indices().prepareRolloverIndex(ILMHistoryStore.ILM_HISTORY_ALIAS).get();
-
-        assertTrue(rolloverResponse.isAcknowledged());
-        assertThat(rolloverResponse.getNewIndex(), is(secondIndex));
-
-        GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(secondIndex).get();
-        assertThat(getIndexResponse.getIndices(), arrayContaining(secondIndex));
-    }
-}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java
index c11eb25f4c08c..1a019710b6c08 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java
@@ -10,10 +10,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ResourceAlreadyExistsException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.alias.Alias;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkProcessor;
@@ -22,10 +19,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.OriginSettingClient;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -33,8 +27,6 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
@@ -42,14 +34,13 @@
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN;
 import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING;
+import static org.elasticsearch.xpack.ilm.history.ILMHistoryTemplateRegistry.ILM_TEMPLATE_NAME;
 import static org.elasticsearch.xpack.ilm.history.ILMHistoryTemplateRegistry.INDEX_TEMPLATE_VERSION;
-import static org.elasticsearch.xpack.ilm.history.ILMHistoryTemplateRegistry.TEMPLATE_ILM_HISTORY;
 
 /**
  * The {@link ILMHistoryStore} handles indexing {@link ILMHistoryItem} documents into the
@@ -59,8 +50,7 @@
 public class ILMHistoryStore implements Closeable {
     private static final Logger logger = LogManager.getLogger(ILMHistoryStore.class);
 
-    public static final String ILM_HISTORY_INDEX_PREFIX = "ilm-history-" + INDEX_TEMPLATE_VERSION + "-";
-    public static final String ILM_HISTORY_ALIAS = "ilm-history-" + INDEX_TEMPLATE_VERSION;
+    public static final String ILM_HISTORY_DATA_STREAM = "ilm-history-" + INDEX_TEMPLATE_VERSION;
 
     private final boolean ilmHistoryEnabled;
     private final BulkProcessor processor;
@@ -75,18 +65,8 @@ public ILMHistoryStore(Settings nodeSettings, Client client, ClusterService clus
             new BulkProcessor.Listener() {
                 @Override
                 public void beforeBulk(long executionId, BulkRequest request) {
-                    // Prior to actually performing the bulk, we should ensure the index exists, and
-                    // if we were unable to create it or it was in a bad state, we should not
-                    // attempt to index documents.
-                    try {
-                        final CompletableFuture<Boolean> indexCreated = new CompletableFuture<>();
-                        ensureHistoryIndex(client, clusterService.state(), ActionListener.wrap(indexCreated::complete,
-                            ex -> {
-                                logger.warn("failed to create ILM history store index prior to issuing bulk request", ex);
-                                indexCreated.completeExceptionally(ex);
-                            }));
-                        indexCreated.get(2, TimeUnit.MINUTES);
-                    } catch (Exception e) {
+                    if (clusterService.state().getMetadata().templatesV2().containsKey(ILM_TEMPLATE_NAME) == false) {
+                        ElasticsearchException e = new ElasticsearchException("no ILM history template");
                         logger.warn(new ParameterizedMessage("unable to index the following ILM history items:\n{}",
                             request.requests().stream()
                                 .filter(dwr -> (dwr instanceof IndexRequest))
@@ -150,10 +130,10 @@ public void putAsync(ILMHistoryItem item) {
                 LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), item);
             return;
         }
-        logger.trace("queueing ILM history item for indexing [{}]: [{}]", ILM_HISTORY_ALIAS, item);
+        logger.trace("queueing ILM history item for indexing [{}]: [{}]", ILM_HISTORY_DATA_STREAM, item);
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
             item.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            IndexRequest request = new IndexRequest(ILM_HISTORY_ALIAS).source(builder);
+            IndexRequest request = new IndexRequest(ILM_HISTORY_DATA_STREAM).source(builder).opType(DocWriteRequest.OpType.CREATE);
             // TODO: remove the threadpool wrapping when the .add call is non-blocking
             //  (it can currently execute the bulk request occasionally)
             //  see: https://github.com/elastic/elasticsearch/issues/50440
@@ -162,83 +142,12 @@ public void putAsync(ILMHistoryItem item) {
                     processor.add(request);
                 } catch (Exception e) {
                     logger.error(new ParameterizedMessage("failed add ILM history item to queue for index [{}]: [{}]",
-                        ILM_HISTORY_ALIAS, item), e);
+                        ILM_HISTORY_DATA_STREAM, item), e);
                 }
             });
         } catch (IOException exception) {
             logger.error(new ParameterizedMessage("failed to queue ILM history item in index [{}]: [{}]",
-                ILM_HISTORY_ALIAS, item), exception);
-        }
-    }
-
-    /**
-     * Checks if the ILM history index exists, and if not, creates it.
-     *
-     * @param client  The client to use to create the index if needed
-     * @param state   The current cluster state, to determine if the alias exists
-     * @param listener Called after the index has been created. `onResponse` called with `true` if the index was created,
-     *                `false` if it already existed.
-     */
-    @SuppressWarnings("unchecked")
-    static void ensureHistoryIndex(Client client, ClusterState state, ActionListener<Boolean> listener) {
-        final String initialHistoryIndexName = ILM_HISTORY_INDEX_PREFIX + "000001";
-        final IndexAbstraction ilmHistory = state.metadata().getIndicesLookup().get(ILM_HISTORY_ALIAS);
-        final IndexAbstraction initialHistoryIndex = state.metadata().getIndicesLookup().get(initialHistoryIndexName);
-
-        if (ilmHistory == null && initialHistoryIndex == null) {
-            // No alias or index exists with the expected names, so create the index with appropriate alias
-            logger.debug("creating ILM history index [{}]", initialHistoryIndexName);
-
-            // Template below should be already defined as real index template but it can be deleted. To avoid race condition with its
-            // recreation we apply settings and mappings ourselves
-            byte[] templateBytes = TEMPLATE_ILM_HISTORY.loadBytes();
-            Map<String, Object> templateAsMap = XContentHelper.convertToMap(new BytesArray(templateBytes, 0, templateBytes.length),
-                false, XContentType.JSON).v2();
-            templateAsMap = (Map<String, Object>) templateAsMap.get("template");
-
-
-            client.admin().indices().prepareCreate(initialHistoryIndexName)
-                .setSettings((Map<String, ?>) templateAsMap.get("settings"))
-                .setMapping((Map<String, Object>) templateAsMap.get("mappings"))
-                .setWaitForActiveShards(1)
-                .addAlias(new Alias(ILM_HISTORY_ALIAS).writeIndex(true).isHidden(true))
-                .execute(new ActionListener<>() {
-                    @Override
-                    public void onResponse(CreateIndexResponse response) {
-                        listener.onResponse(true);
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        if (e instanceof ResourceAlreadyExistsException) {
-                            // The index didn't exist before we made the call, there was probably a race - just ignore this
-                            logger.debug("index [{}] was created after checking for its existence, likely due to a concurrent call",
-                                initialHistoryIndexName);
-                            listener.onResponse(false);
-                        } else {
-                            listener.onFailure(e);
-                        }
-                    }
-                });
-        } else if (ilmHistory == null) {
-            // alias does not exist but initial index does, something is broken
-            listener.onFailure(new IllegalStateException("ILM history index [" + initialHistoryIndexName +
-                "] already exists but does not have alias [" + ILM_HISTORY_ALIAS + "]"));
-        } else if (ilmHistory.getType() == IndexAbstraction.Type.ALIAS) {
-            if (ilmHistory.getWriteIndex() != null) {
-                // The alias exists and has a write index, so we're good
-                listener.onResponse(false);
-            } else {
-                // The alias does not have a write index, so we can't index into it
-                listener.onFailure(new IllegalStateException("ILM history alias [" + ILM_HISTORY_ALIAS + "does not have a write index"));
-            }
-        } else if (ilmHistory.getType() != IndexAbstraction.Type.ALIAS) {
-            // This is not an alias, error out
-            listener.onFailure(new IllegalStateException("ILM history alias [" + ILM_HISTORY_ALIAS +
-                "] already exists as " + ilmHistory.getType().getDisplayName()));
-        } else {
-            logger.error("unexpected IndexOrAlias for [{}]: [{}]", ILM_HISTORY_ALIAS, ilmHistory);
-            assert false : ILM_HISTORY_ALIAS + " cannot be both an alias and not an alias simultaneously";
+                ILM_HISTORY_DATA_STREAM, item), exception);
         }
     }
 
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java
index c8835acbd0b39..a8819d2801dbc 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java
@@ -30,7 +30,8 @@ public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry {
     // version 2: convert to hidden index
     // version 3: templates moved to composable templates
     // version 4: add `allow_auto_create` setting
-    public static final int INDEX_TEMPLATE_VERSION = 4;
+    // version 5: convert to data stream
+    public static final int INDEX_TEMPLATE_VERSION = 5;
 
     public static final String ILM_TEMPLATE_VERSION_VARIABLE = "xpack.ilm_history.template.version";
     public static final String ILM_TEMPLATE_NAME = "ilm-history";
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java
index f22395069d608..e256b0fdcd048 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java
@@ -6,14 +6,11 @@
 
 package org.elasticsearch.xpack.ilm.history;
 
-import org.elasticsearch.ResourceAlreadyExistsException;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
@@ -23,36 +20,37 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.AliasMetadata;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
-import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ilm.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
 import org.hamcrest.Matchers;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING;
-import static org.elasticsearch.xpack.ilm.history.ILMHistoryStore.ILM_HISTORY_ALIAS;
-import static org.elasticsearch.xpack.ilm.history.ILMHistoryStore.ILM_HISTORY_INDEX_PREFIX;
+import static org.elasticsearch.xpack.ilm.history.ILMHistoryStore.ILM_HISTORY_DATA_STREAM;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -68,9 +66,26 @@ public void setup() {
         threadPool = new TestThreadPool(this.getClass().getName());
         client = new VerifyingClient(threadPool);
         clusterService = ClusterServiceUtils.createClusterService(threadPool);
+        ILMHistoryTemplateRegistry registry = new ILMHistoryTemplateRegistry(clusterService.getSettings(), clusterService, threadPool,
+            client, NamedXContentRegistry.EMPTY);
+        Map<String, ComposableIndexTemplate> templates =
+            registry.getComposableTemplateConfigs().stream().collect(Collectors.toMap(IndexTemplateConfig::getTemplateName,
+                this::parseIndexTemplate));
+        ClusterState state = clusterService.state();
+        ClusterServiceUtils.setState(clusterService,
+            ClusterState.builder(state).metadata(Metadata.builder(state.metadata()).indexTemplates(templates)).build());
         historyStore = new ILMHistoryStore(Settings.EMPTY, client, clusterService, threadPool);
     }
 
+    private ComposableIndexTemplate parseIndexTemplate(IndexTemplateConfig c) {
+        try {
+            return ComposableIndexTemplate.parse(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION, c.loadBytes()));
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
     @After
     public void setdown() {
         historyStore.close();
@@ -109,14 +124,11 @@ public void testPut() throws Exception {
 
             AtomicInteger calledTimes = new AtomicInteger(0);
             client.setVerifier((action, request, listener) -> {
-                if (action instanceof CreateIndexAction && request instanceof CreateIndexRequest) {
-                    return new CreateIndexResponse(true, true, ((CreateIndexRequest) request).index());
-                }
                 calledTimes.incrementAndGet();
                 assertThat(action, instanceOf(BulkAction.class));
                 assertThat(request, instanceOf(BulkRequest.class));
                 BulkRequest bulkRequest = (BulkRequest) request;
-                bulkRequest.requests().forEach(dwr -> assertEquals(ILM_HISTORY_ALIAS, dwr.index()));
+                bulkRequest.requests().forEach(dwr -> assertEquals(ILM_HISTORY_DATA_STREAM, dwr.index()));
                 assertNotNull(listener);
 
                 // The content of this BulkResponse doesn't matter, so just make it have the same number of responses
@@ -150,7 +162,7 @@ public void testPut() throws Exception {
                 assertThat(request, instanceOf(BulkRequest.class));
                 BulkRequest bulkRequest = (BulkRequest) request;
                 bulkRequest.requests().forEach(dwr -> {
-                    assertEquals(ILM_HISTORY_ALIAS, dwr.index());
+                    assertEquals(ILM_HISTORY_DATA_STREAM, dwr.index());
                     assertThat(dwr, instanceOf(IndexRequest.class));
                     IndexRequest ir = (IndexRequest) dwr;
                     String indexedDocument = ir.source().utf8ToString();
@@ -173,197 +185,6 @@ public void testPut() throws Exception {
         }
     }
 
-    public void testHistoryIndexNeedsCreation() throws InterruptedException {
-        ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5)))
-            .metadata(Metadata.builder())
-            .build();
-
-        client.setVerifier((a, r, l) -> {
-            assertThat(a, instanceOf(CreateIndexAction.class));
-            assertThat(r, instanceOf(CreateIndexRequest.class));
-            CreateIndexRequest request = (CreateIndexRequest) r;
-            assertThat(request.aliases(), Matchers.hasSize(1));
-            request.aliases().forEach(alias -> {
-                assertThat(alias.name(), equalTo(ILM_HISTORY_ALIAS));
-                assertTrue(alias.writeIndex());
-            });
-            return new CreateIndexResponse(true, true, request.index());
-        });
-
-        CountDownLatch latch = new CountDownLatch(1);
-        ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap(
-            Assert::assertTrue,
-            ex -> {
-                logger.error(ex);
-                fail("should have called onResponse, not onFailure");
-            }), latch));
-
-        ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS);
-    }
-
-    public void testHistoryIndexProperlyExistsAlready() throws InterruptedException {
-        ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5)))
-            .metadata(Metadata.builder()
-                .put(IndexMetadata.builder(ILM_HISTORY_INDEX_PREFIX + "000001")
-                    .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT))
-                    .numberOfShards(randomIntBetween(1,10))
-                    .numberOfReplicas(randomIntBetween(1,10))
-                    .putAlias(AliasMetadata.builder(ILM_HISTORY_ALIAS)
-                        .writeIndex(true)
-                        .build())))
-            .build();
-
-        client.setVerifier((a, r, l) -> {
-            fail("no client calls should have been made");
-            return null;
-        });
-
-        CountDownLatch latch = new CountDownLatch(1);
-        ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap(
-            Assert::assertFalse,
-            ex -> {
-                logger.error(ex);
-                fail("should have called onResponse, not onFailure");
-            }), latch));
-
-        ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS);
-    }
-
-    public void testHistoryIndexHasNoWriteIndex() throws InterruptedException {
-        ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5)))
-            .metadata(Metadata.builder()
-                .put(IndexMetadata.builder(ILM_HISTORY_INDEX_PREFIX + "000001")
-                    .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT))
-                    .numberOfShards(randomIntBetween(1,10))
-                    .numberOfReplicas(randomIntBetween(1,10))
-                    .putAlias(AliasMetadata.builder(ILM_HISTORY_ALIAS)
-                        .build()))
-                .put(IndexMetadata.builder(randomAlphaOfLength(5))
-                    .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT))
-                    .numberOfShards(randomIntBetween(1,10))
-                    .numberOfReplicas(randomIntBetween(1,10))
-                    .putAlias(AliasMetadata.builder(ILM_HISTORY_ALIAS)
-                        .build())))
-            .build();
-
-        client.setVerifier((a, r, l) -> {
-            fail("no client calls should have been made");
-            return null;
-        });
-
-        CountDownLatch latch = new CountDownLatch(1);
-        ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap(
-            indexCreated -> fail("should have called onFailure, not onResponse"),
-            ex -> {
-                assertThat(ex, instanceOf(IllegalStateException.class));
-                assertThat(ex.getMessage(), Matchers.containsString("ILM history alias [" + ILM_HISTORY_ALIAS +
-                    "does not have a write index"));
-            }), latch));
-
-        ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS);
-    }
-
-    public void testHistoryIndexNotAlias() throws InterruptedException {
-        ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5)))
-            .metadata(Metadata.builder()
-                .put(IndexMetadata.builder(ILM_HISTORY_ALIAS)
-                    .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT))
-                    .numberOfShards(randomIntBetween(1,10))
-                    .numberOfReplicas(randomIntBetween(1,10))))
-            .build();
-
-        client.setVerifier((a, r, l) -> {
-            fail("no client calls should have been made");
-            return null;
-        });
-
-        CountDownLatch latch = new CountDownLatch(1);
-        ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap(
-            indexCreated -> fail("should have called onFailure, not onResponse"),
-            ex -> {
-                assertThat(ex, instanceOf(IllegalStateException.class));
-                assertThat(ex.getMessage(), Matchers.containsString("ILM history alias [" + ILM_HISTORY_ALIAS +
-                    "] already exists as concrete index"));
-            }), latch));
-
-        ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS);
-    }
-
-    public void testHistoryIndexCreatedConcurrently() throws InterruptedException {
-        ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5)))
-            .metadata(Metadata.builder())
-            .build();
-
-        client.setVerifier((a, r, l) -> {
-            assertThat(a, instanceOf(CreateIndexAction.class));
-            assertThat(r, instanceOf(CreateIndexRequest.class));
-            CreateIndexRequest request = (CreateIndexRequest) r;
-            assertThat(request.aliases(), Matchers.hasSize(1));
-            request.aliases().forEach(alias -> {
-                assertThat(alias.name(), equalTo(ILM_HISTORY_ALIAS));
-                assertTrue(alias.writeIndex());
-            });
-            throw new ResourceAlreadyExistsException("that index already exists");
-        });
-
-        CountDownLatch latch = new CountDownLatch(1);
-        ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap(
-            Assert::assertFalse,
-            ex -> {
-                logger.error(ex);
-                fail("should have called onResponse, not onFailure");
-            }), latch));
-
-        ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS);
-    }
-
-    public void testHistoryAliasDoesntExistButIndexDoes() throws InterruptedException {
-        final String initialIndex = ILM_HISTORY_INDEX_PREFIX + "000001";
-        ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5)))
-            .metadata(Metadata.builder()
-                .put(IndexMetadata.builder(initialIndex)
-                    .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT))
-                    .numberOfShards(randomIntBetween(1,10))
-                    .numberOfReplicas(randomIntBetween(1,10))))
-            .build();
-
-        client.setVerifier((a, r, l) -> {
-            fail("no client calls should have been made");
-            return null;
-        });
-
-        CountDownLatch latch = new CountDownLatch(1);
-        ILMHistoryStore.ensureHistoryIndex(client, state, new LatchedActionListener<>(ActionListener.wrap(
-            response -> {
-                logger.error(response);
-                fail("should have called onFailure, not onResponse");
-            },
-            ex -> {
-                assertThat(ex, instanceOf(IllegalStateException.class));
-                assertThat(ex.getMessage(), Matchers.containsString("ILM history index [" + initialIndex +
-                    "] already exists but does not have alias [" + ILM_HISTORY_ALIAS + "]"));
-            }), latch));
-
-        ElasticsearchAssertions.awaitLatch(latch, 10, TimeUnit.SECONDS);
-    }
-
-    @SuppressWarnings("unchecked")
-    private void assertContainsMap(String indexedDocument, Map<String, Object> map) {
-        map.forEach((k, v) -> {
-            assertThat(indexedDocument, Matchers.containsString(k));
-            if (v instanceof Map) {
-                assertContainsMap(indexedDocument, (Map<String, Object>) v);
-            }
-            if (v instanceof Iterable) {
-                ((Iterable) v).forEach(elem -> {
-                    assertThat(indexedDocument, Matchers.containsString(elem.toString()));
-                });
-            } else {
-                assertThat(indexedDocument, Matchers.containsString(v.toString()));
-            }
-        });
-    }
-
     /**
      * A client that delegates to a verifying function for action/request/listener
      */

From c4e3fc45f0a781d528c198866ee3de178f562991 Mon Sep 17 00:00:00 2001
From: Andras Palinkas <andras.palinkas@elastic.co>
Date: Mon, 9 Nov 2020 17:41:11 -0500
Subject: [PATCH 27/34] SQL: Remove the defensive copy from the AttributeMap
 (#64830)

Remove the defensive copy from the builder.
---
 .../xpack/ql/expression/AttributeMap.java     | 22 ++++---------------
 .../ql/expression/AttributeMapTests.java      |  9 --------
 2 files changed, 4 insertions(+), 27 deletions(-)

diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java
index aec6102c6d906..56bc692a4767a 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/AttributeMap.java
@@ -364,36 +364,22 @@ public static <E> Builder<E> builder(AttributeMap<E> map) {
     }
 
     public static class Builder<E> {
-        private AttributeMap<E> map = null;
-        private AttributeMap<E> previouslyBuiltMap = null;
+        private AttributeMap<E> map = new AttributeMap<>();
 
         private Builder() {}
 
-        private AttributeMap<E> map() {
-            if (map == null) {
-                map = new AttributeMap<>();
-                if (previouslyBuiltMap != null) {
-                    map.addAll(previouslyBuiltMap);
-                }
-            }
-            return map;
-        }
-
         public Builder<E> put(Attribute attr, E value) {
-            map().add(attr, value);
+            map.add(attr, value);
             return this;
         }
 
         public Builder<E> putAll(AttributeMap<E> m) {
-            map().addAll(m);
+            map.addAll(m);
             return this;
         }
 
         public AttributeMap<E> build() {
-            AttributeMap<E> m = map();
-            previouslyBuiltMap = m;
-            map = null;
-            return m;
+            return map;
         }
     }
 }
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java
index 298ade4db11da..9f5d93beb6670 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/AttributeMapTests.java
@@ -90,15 +90,6 @@ public void testBuilder() {
         assertThat(m.containsValue("on"), is(false));
         assertThat(m.attributeNames(), contains("one", "two", "three"));
         assertThat(m.values(), contains("one", "two", "three"));
-
-        // defensive copying
-        builder.put(a("four"), "four");
-        AttributeMap<String> m2 = builder.build();
-        assertThat(m.size(), is(3));
-        assertThat(m.isEmpty(), is(false));
-        assertThat(m2.size(), is(4));
-        assertThat(m.isEmpty(), is(false));
-        assertThat(m2.attributeNames(), contains("one", "two", "three", "four"));
     }
 
     public void testSingleItemConstructor() {

From 996b49cbf9e88761d0832f1c38eaa03c997a68cc Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 9 Nov 2020 16:59:42 -0500
Subject: [PATCH 28/34] Add TODO for IndexWriter#flushNextBuffer

Relates #34553
---
 .../java/org/elasticsearch/index/engine/InternalEngine.java     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index db5ac0db99da8..ed76f294ca54e 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -1656,6 +1656,8 @@ final boolean refresh(String source, SearcherScope scope, boolean block) throws
 
     @Override
     public void writeIndexingBuffer() throws EngineException {
+        // TODO: revise https://github.com/elastic/elasticsearch/pull/34553 to use IndexWriter.flushNextBuffer to flush only the largest
+        // pending DWPT. Note that benchmarking this PR with a heavy update user case (geonames) and a small heap (1GB) caused OOM.
         refresh("write indexing buffer", SearcherScope.INTERNAL, false);
     }
 

From 3f3c874b63e9fc15ff7b05421753ca9142bd1fc1 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Mon, 9 Nov 2020 15:12:07 -0800
Subject: [PATCH 29/34] Remove unnecessary classloader permissions in xpack
 (#64750)

Xpack core contains a utility class for setting and restoring the
context class loader when entering certain libraries which depend on the
context classloader. The permissions for getting and setting the
classloader are only needed within this core class,
RestorableContextClassLoader, yet thes permissions (along with the
comment) have been copied to a few other security policy files. This
commit removes those unnecessary permissions from modules outside of
xpack core.
---
 .../core/security/support/RestorableContextClassLoader.java | 6 +++++-
 .../src/main/plugin-metadata/plugin-security.policy         | 4 ----
 .../src/main/plugin-metadata/plugin-security.policy         | 4 ----
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java
index f3c7ec44a5249..7d1e0fd101b3d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RestorableContextClassLoader.java
@@ -22,7 +22,11 @@ public class RestorableContextClassLoader implements AutoCloseable {
     private ClassLoader restore;
 
     public RestorableContextClassLoader(Class<?> fromClass) throws PrivilegedActionException {
-        this(Thread.currentThread(), fromClass.getClassLoader());
+        this(Thread.currentThread(), getClassLoader(fromClass));
+    }
+
+    private static ClassLoader getClassLoader(Class<?> fromClass) throws PrivilegedActionException {
+        return AccessController.doPrivileged((PrivilegedExceptionAction<ClassLoader>) fromClass::getClassLoader);
     }
 
     public RestorableContextClassLoader(Thread thread, ClassLoader setClassLoader) throws PrivilegedActionException {
diff --git a/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
index 65334a8db5760..9013d541e4546 100644
--- a/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
@@ -4,10 +4,6 @@ grant {
   // ApacheXMLSecurityInitializer
   permission java.util.PropertyPermission "org.apache.xml.security.ignoreLineBreaks", "read,write";
 
-  // needed because of SAML (cf. o.e.x.c.s.s.RestorableContextClassLoader)
-  permission java.lang.RuntimePermission "getClassLoader";
-  permission java.lang.RuntimePermission "setContextClassLoader";
-
   // needed during initialization of OpenSAML library where xml security algorithms are registered
   // see https://github.com/apache/santuario-java/blob/e79f1fe4192de73a975bc7246aee58ed0703343d/src/main/java/org/apache/xml/security/utils/JavaUtils.java#L205-L220
   // and https://git.shibboleth.net/view/?p=java-opensaml.git;a=blob;f=opensaml-xmlsec-impl/src/main/java/org/opensaml/xmlsec/signature/impl/SignatureMarshaller.java;hb=db0eaa64210f0e32d359cd6c57bedd57902bf811#l52
diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
index 836653c56fbc5..50057b5f839d8 100644
--- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
@@ -4,10 +4,6 @@ grant {
   // needed because of problems in unbound LDAP library
   permission java.util.PropertyPermission "*", "read,write";
 
-  // needed because of SAML (cf. o.e.x.c.s.s.RestorableContextClassLoader)
-  permission java.lang.RuntimePermission "getClassLoader";
-  permission java.lang.RuntimePermission "setContextClassLoader";
-
   // needed during initialization of OpenSAML library where xml security algorithms are registered
   // see https://github.com/apache/santuario-java/blob/e79f1fe4192de73a975bc7246aee58ed0703343d/src/main/java/org/apache/xml/security/utils/JavaUtils.java#L205-L220
   // and https://git.shibboleth.net/view/?p=java-opensaml.git;a=blob;f=opensaml-xmlsec-impl/src/main/java/org/opensaml/xmlsec/signature/impl/SignatureMarshaller.java;hb=db0eaa64210f0e32d359cd6c57bedd57902bf811#l52

From c212873e8ac9c9542b3bd215584ec862df2f3971 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Mon, 9 Nov 2020 15:12:58 -0800
Subject: [PATCH 30/34] Remove provider permissions from ingest-attachment
 plugin (#64814)

At some point the the past, PDFBox setup the bouncycastle security
provider explicitly. However, since then it looks like they have moved
away from that and no longer force that provider
(https://issues.apache.org/jira/browse/PDFBOX-2963), which means we no
longer need to grant those security provider permissions. This commit
removes these legacy permissions.
---
 .../src/main/plugin-metadata/plugin-security.policy            | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy b/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy
index bcc5eef3193d7..56391a9ab5280 100644
--- a/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy
@@ -22,9 +22,6 @@ grant {
   // needed to apply additional sandboxing to tika parsing
   permission java.security.SecurityPermission "createAccessControlContext";
 
-  // TODO: fix PDFBox not to actually install bouncy castle like this
-  permission java.security.SecurityPermission "putProviderProperty.BC";
-  permission java.security.SecurityPermission "insertProvider";
   // TODO: fix POI XWPF to not do this: https://bz.apache.org/bugzilla/show_bug.cgi?id=58597
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   // needed by xmlbeans, as part of POI for MS xml docs

From f5598475fa53219fe226b36b50e768ace75886c4 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Mon, 9 Nov 2020 15:58:54 -0800
Subject: [PATCH 31/34] Treat all xpack as modules (#64832)

When rest tests install plugins, they currently treat xpack as plugins
instead of modules. This was a side effect of splitting the rest test
plugin out from PluginBuildPlugin. However, that means xpack modules are
being run through the elasticsearch plugin installer, which is not
guaranteed to work. This commit reworks rest tests to treat anything
under xpack as a module. A side effect of this is that QA plugins under
xpack get treated as modules. This is ok because they are just for
testing, we don't need to validate them in the same way we do actual
plugins. Additionally, this commit relaxes the expection that modules
are only added for the integ test distribution. There is already a check
to not overwrite an existing module, so this wasn't a useful
optimization anyways.
---
 .../gradle/test/rest/RestTestUtil.java        |  4 +--
 .../testclusters/ElasticsearchNode.java       | 36 +++++++++----------
 .../plugin/async-search/qa/rest/build.gradle  |  1 +
 3 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java
index 1667d674870e8..60ae306495800 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java
@@ -62,10 +62,10 @@ static Provider<RestIntegTestTask> registerTask(Project project, SourceSet sourc
             project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
                 Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
                 testTask.dependsOn(bundle);
-                if (project.getPath().contains("modules:")) {
+                if (project.getPath().contains("modules:") || project.getPath().startsWith(":x-pack:plugin")) {
                     testTask.getClusters().forEach(c -> c.module(bundle.getArchiveFile()));
                 } else {
-                    testTask.getClusters().forEach(c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile())));
+                    testTask.getClusters().forEach(c -> c.plugin(bundle.getArchiveFile()));
                 }
             });
         });
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
index c946ea0db7872..327e9a753f8bb 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
@@ -617,27 +617,23 @@ private void copyExtraJars() {
     }
 
     private void installModules() {
-        if (testDistribution == TestDistribution.INTEG_TEST) {
-            logToProcessStdout("Installing " + modules.size() + "modules");
-            for (Provider<File> module : modules) {
-                Path destination = getDistroDir().resolve("modules")
-                    .resolve(module.get().getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", ""));
-                // only install modules that are not already bundled with the integ-test distribution
-                if (Files.exists(destination) == false) {
-                    fileSystemOperations.copy(spec -> {
-                        if (module.get().getName().toLowerCase().endsWith(".zip")) {
-                            spec.from(archiveOperations.zipTree(module));
-                        } else if (module.get().isDirectory()) {
-                            spec.from(module);
-                        } else {
-                            throw new IllegalArgumentException("Not a valid module " + module + " for " + this);
-                        }
-                        spec.into(destination);
-                    });
-                }
+        logToProcessStdout("Installing " + modules.size() + "modules");
+        for (Provider<File> module : modules) {
+            Path destination = getDistroDir().resolve("modules")
+                .resolve(module.get().getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", ""));
+            // only install modules that are not already bundled with the integ-test distribution
+            if (Files.exists(destination) == false) {
+                fileSystemOperations.copy(spec -> {
+                    if (module.get().getName().toLowerCase().endsWith(".zip")) {
+                        spec.from(archiveOperations.zipTree(module));
+                    } else if (module.get().isDirectory()) {
+                        spec.from(module);
+                    } else {
+                        throw new IllegalArgumentException("Not a valid module " + module + " for " + this);
+                    }
+                    spec.into(destination);
+                });
             }
-        } else {
-            LOGGER.info("Not installing " + modules.size() + "(s) since the " + distributions + " distribution already has them");
         }
     }
 
diff --git a/x-pack/plugin/async-search/qa/rest/build.gradle b/x-pack/plugin/async-search/qa/rest/build.gradle
index 5f3f810fa45bf..97f7ec4d789e3 100644
--- a/x-pack/plugin/async-search/qa/rest/build.gradle
+++ b/x-pack/plugin/async-search/qa/rest/build.gradle
@@ -2,6 +2,7 @@ apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.yaml-rest-test'
 
 esplugin {
+  name 'test-deprecated-query'
   description 'Deprecated query plugin'
   classname 'org.elasticsearch.query.DeprecatedQueryPlugin'
 }

From 23232c13917071e9d2b9ffbba41a9d46e47d82ea Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@elastic.co>
Date: Mon, 9 Nov 2020 16:20:12 -0800
Subject: [PATCH 32/34] Remove unused permissions from xpack module security
 policies (#64747)

When xpack was once a single module, all security permissions had to be
defined together. Since it was split into multiple modules, new modules
have often use a copy/paste approach to the security policy of that new
file. Yet most of those permissions are not needed in the new modules.
This commit cleans up a particularly prevalent set of permission grants.
These are completely unused, and don't even resolve when the policy is
parsed, because these netty and rest client codebases do not exist in
the context of these xpack modules.
---
 .../plugin-metadata/plugin-security.policy    |  5 ----
 .../plugin-metadata/plugin-security.policy    | 21 -----------------
 .../plugin-metadata/plugin-security.policy    | 21 -----------------
 .../plugin-metadata/plugin-security.policy    | 10 --------
 .../plugin-metadata/plugin-security.policy    | 21 -----------------
 .../plugin-metadata/plugin-security.policy    | 23 -------------------
 .../plugin-metadata/plugin-security.policy    | 16 -------------
 .../plugin-metadata/plugin-security.policy    | 21 -----------------
 .../plugin-metadata/plugin-security.policy    | 21 -----------------
 9 files changed, 159 deletions(-)

diff --git a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
index 964058120f9f4..a0cd3bc741b29 100644
--- a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy
@@ -23,11 +23,6 @@ grant codeBase "${codebase.netty-transport}" {
    permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
 };
 
-grant codeBase "${codebase.elasticsearch-rest-client}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
-
 grant codeBase "${codebase.httpasyncclient}" {
   // rest client uses system properties which gets the default proxy
   permission java.net.NetPermission "getProxySelector";
diff --git a/x-pack/plugin/deprecation/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/deprecation/src/main/plugin-metadata/plugin-security.policy
index f603bf9ad63ba..16701ab74d8c9 100644
--- a/x-pack/plugin/deprecation/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/deprecation/src/main/plugin-metadata/plugin-security.policy
@@ -2,24 +2,3 @@ grant {
   // needed for multiple server implementations used in tests
   permission java.net.SocketPermission "*", "accept,connect";
 };
-
-grant codeBase "${codebase.netty-common}" {
-   // for reading the system-wide configuration for the backlog of established sockets
-   permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
-};
-
-grant codeBase "${codebase.netty-transport}" {
-   // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
-   // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
-   permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
-};
-
-grant codeBase "${codebase.elasticsearch-rest-client}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
-
-grant codeBase "${codebase.httpasyncclient}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
\ No newline at end of file
diff --git a/x-pack/plugin/graph/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/graph/src/main/plugin-metadata/plugin-security.policy
index f603bf9ad63ba..16701ab74d8c9 100644
--- a/x-pack/plugin/graph/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/graph/src/main/plugin-metadata/plugin-security.policy
@@ -2,24 +2,3 @@ grant {
   // needed for multiple server implementations used in tests
   permission java.net.SocketPermission "*", "accept,connect";
 };
-
-grant codeBase "${codebase.netty-common}" {
-   // for reading the system-wide configuration for the backlog of established sockets
-   permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
-};
-
-grant codeBase "${codebase.netty-transport}" {
-   // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
-   // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
-   permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
-};
-
-grant codeBase "${codebase.elasticsearch-rest-client}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
-
-grant codeBase "${codebase.httpasyncclient}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
\ No newline at end of file
diff --git a/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
index 9013d541e4546..42baf37c00ba8 100644
--- a/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/identity-provider/src/main/plugin-metadata/plugin-security.policy
@@ -13,13 +13,3 @@ grant {
   // needed for multiple server implementations used in tests
   permission java.net.SocketPermission "*", "accept,connect";
 };
-
-grant codeBase "${codebase.elasticsearch-rest-client}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
-
-grant codeBase "${codebase.httpasyncclient}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
diff --git a/x-pack/plugin/logstash/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/logstash/src/main/plugin-metadata/plugin-security.policy
index c54f07cf5cd8d..16701ab74d8c9 100644
--- a/x-pack/plugin/logstash/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/logstash/src/main/plugin-metadata/plugin-security.policy
@@ -2,24 +2,3 @@ grant {
   // needed for multiple server implementations used in tests
   permission java.net.SocketPermission "*", "accept,connect";
 };
-
-grant codeBase "${codebase.netty-common}" {
-   // for reading the system-wide configuration for the backlog of established sockets
-   permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
-};
-
-grant codeBase "${codebase.netty-transport}" {
-   // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
-   // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
-   permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
-};
-
-grant codeBase "${codebase.elasticsearch-rest-client}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
-
-grant codeBase "${codebase.httpasyncclient}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
diff --git a/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy
index 74f27ed286eb6..b23580c25c08f 100644
--- a/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy
@@ -5,26 +5,3 @@ grant {
   // needed for Windows named pipes in machine learning
   permission java.io.FilePermission "\\\\.\\pipe\\*", "read,write";
 };
-
-grant codeBase "${codebase.netty-common}" {
-   // for reading the system-wide configuration for the backlog of established sockets
-   permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
-   // Netty sets custom classloader for some of its internal threads
-   permission java.lang.RuntimePermission "setContextClassLoader";
-};
-
-grant codeBase "${codebase.netty-transport}" {
-   // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
-   // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
-   permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
-};
-
-grant codeBase "${codebase.elasticsearch-rest-client}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
-
-grant codeBase "${codebase.httpasyncclient}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
diff --git a/x-pack/plugin/monitoring/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/monitoring/src/main/plugin-metadata/plugin-security.policy
index beb104a6b3d09..ef079a5c16e46 100644
--- a/x-pack/plugin/monitoring/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/monitoring/src/main/plugin-metadata/plugin-security.policy
@@ -17,23 +17,7 @@ grant {
   permission java.net.SocketPermission "*", "accept,connect";
 };
 
-grant codeBase "${codebase.netty-common}" {
-   // for reading the system-wide configuration for the backlog of established sockets
-   permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
-};
-
-grant codeBase "${codebase.netty-transport}" {
-   // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
-   // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
-   permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
-};
-
 grant codeBase "${codebase.elasticsearch-rest-client}" {
   // rest client uses system properties which gets the default proxy
   permission java.net.NetPermission "getProxySelector";
 };
-
-grant codeBase "${codebase.httpasyncclient}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
\ No newline at end of file
diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
index 50057b5f839d8..3756275fb2e1c 100644
--- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy
@@ -32,24 +32,3 @@ grant {
   permission java.lang.RuntimePermission "accessUserInformation";
   permission java.lang.RuntimePermission "getFileStoreAttributes";
 };
-
-grant codeBase "${codebase.netty-common}" {
-   // for reading the system-wide configuration for the backlog of established sockets
-   permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
-};
-
-grant codeBase "${codebase.netty-transport}" {
-   // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
-   // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
-   permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
-};
-
-grant codeBase "${codebase.elasticsearch-rest-client}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
-
-grant codeBase "${codebase.httpasyncclient}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
diff --git a/x-pack/plugin/watcher/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/watcher/src/main/plugin-metadata/plugin-security.policy
index 8472a42a64832..d27ded771b86f 100644
--- a/x-pack/plugin/watcher/src/main/plugin-metadata/plugin-security.policy
+++ b/x-pack/plugin/watcher/src/main/plugin-metadata/plugin-security.policy
@@ -13,24 +13,3 @@ grant {
   // needed for multiple server implementations used in tests
   permission java.net.SocketPermission "*", "accept,connect";
 };
-
-grant codeBase "${codebase.netty-common}" {
-   // for reading the system-wide configuration for the backlog of established sockets
-   permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
-};
-
-grant codeBase "${codebase.netty-transport}" {
-   // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
-   // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
-   permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
-};
-
-grant codeBase "${codebase.elasticsearch-rest-client}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
-
-grant codeBase "${codebase.httpasyncclient}" {
-  // rest client uses system properties which gets the default proxy
-  permission java.net.NetPermission "getProxySelector";
-};
\ No newline at end of file

From 33b408ffc931399255b822126746edbd80d14ae7 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 9 Nov 2020 19:38:47 -0500
Subject: [PATCH 33/34] Realtime get from in-memory segment when possible
 (#64504)

If the reader wrapper is specified, then we can't perform a realtime get
using operations from translog. With this change, we will create an
in-memory Lucene segment from that indexing operation and perform a
realtime get from that segment to avoid refresh storms.
---
 .../org/elasticsearch/get/GetActionIT.java    |  15 +-
 .../elasticsearch/index/engine/Engine.java    |  13 +-
 .../index/engine/InternalEngine.java          |  44 ++-
 .../index/engine/ReadOnlyEngine.java          |   6 +-
 .../engine/SingleDocDirectoryReader.java      | 278 ++++++++++++++++++
 .../elasticsearch/index/shard/IndexShard.java |   2 +-
 .../index/engine/InternalEngineTests.java     | 116 +++++---
 .../index/engine/ReadOnlyEngineTests.java     |   2 +-
 .../index/shard/RefreshListenersTests.java    |   4 +-
 .../index/shard/ShardGetServiceTests.java     |   3 +-
 .../index/engine/EngineTestCase.java          |  92 ++++++
 .../index/shard/IndexShardTestCase.java       |   3 +
 .../index/shard/SearcherHelper.java           |  39 +++
 13 files changed, 555 insertions(+), 62 deletions(-)
 create mode 100644 server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/index/shard/SearcherHelper.java

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java
index 9f324d1a5c8b5..11cb5aa77a47c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java
@@ -37,6 +37,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.index.engine.EngineTestCase;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
@@ -47,6 +49,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import static java.util.Collections.singleton;
@@ -65,7 +68,17 @@ public class GetActionIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(InternalSettingsPlugin.class);
+        return List.of(InternalSettingsPlugin.class, SearcherWrapperPlugin.class);
+    }
+
+    public static class SearcherWrapperPlugin extends Plugin {
+        @Override
+        public void onIndexModule(IndexModule indexModule) {
+            super.onIndexModule(indexModule);
+            if (randomBoolean()) {
+                indexModule.setReaderWrapper(indexService -> EngineTestCase.randomReaderWrapper());
+            }
+        }
     }
 
     public void testSimpleGet() {
diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index 3079a05abc330..4042203a4fa05 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -59,6 +59,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ReleasableLock;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapping;
@@ -96,7 +97,6 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.stream.Stream;
 
@@ -559,9 +559,7 @@ public static class NoOpResult extends Result {
 
     }
 
-    protected final GetResult getFromSearcher(Get get, BiFunction<String, SearcherScope, Engine.Searcher> searcherFactory,
-                                                SearcherScope scope) throws EngineException {
-        final Engine.Searcher searcher = searcherFactory.apply("get", scope);
+    protected final GetResult getFromSearcher(Get get, Engine.Searcher searcher) throws EngineException {
         final DocIdAndVersion docIdAndVersion;
         try {
             docIdAndVersion = VersionsAndSeqNoResolver.loadDocIdAndVersion(searcher.getIndexReader(), get.uid(), true);
@@ -596,7 +594,7 @@ protected final GetResult getFromSearcher(Get get, BiFunction<String, SearcherSc
         }
     }
 
-    public abstract GetResult get(Get get, BiFunction<String, SearcherScope, Searcher> searcherFactory) throws EngineException;
+    public abstract GetResult get(Get get, DocumentMapper mapper, Function<Engine.Searcher, Engine.Searcher> searcherWrapper);
 
     /**
      * Acquires a point-in-time reader that can be used to create {@link Engine.Searcher}s on demand.
@@ -1616,6 +1614,7 @@ private GetResult(boolean exists, long version, DocIdAndVersion docIdAndVersion,
             this.docIdAndVersion = docIdAndVersion;
             this.searcher = searcher;
             this.fromTranslog = fromTranslog;
+            assert fromTranslog == false || searcher.getIndexReader() instanceof TranslogLeafReader;
         }
 
         public GetResult(Engine.Searcher searcher, DocIdAndVersion docIdAndVersion, boolean fromTranslog) {
@@ -1630,6 +1629,10 @@ public long version() {
             return this.version;
         }
 
+        /**
+         * Returns {@code true} iff the get was performed from a translog operation. Notes that this returns {@code false}
+         * if the get was performed on an in-memory Lucene segment created from the corresponding translog operation.
+         */
         public boolean isFromTranslog() {
             return fromTranslog;
         }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index ed76f294ca54e..0daf3d1183f91 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -73,6 +73,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.fieldvisitor.IdOnlyFieldVisitor;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.ParseContext;
@@ -618,14 +619,34 @@ private ExternalReaderManager createReaderManager(RefreshWarmerListener external
         }
     }
 
+    private GetResult getFromTranslog(Get get, Translog.Index index, DocumentMapper mapper,
+                                      Function<Searcher, Searcher> searcherWrapper) throws IOException {
+        assert get.isReadFromTranslog();
+        final SingleDocDirectoryReader inMemoryReader = new SingleDocDirectoryReader(shardId, index, mapper, config().getAnalyzer());
+        final Engine.Searcher searcher = new Engine.Searcher("realtime_get", ElasticsearchDirectoryReader.wrap(inMemoryReader, shardId),
+            config().getSimilarity(), config().getQueryCache(), config().getQueryCachingPolicy(), inMemoryReader);
+        final Searcher wrappedSearcher = searcherWrapper.apply(searcher);
+        if (wrappedSearcher == searcher) {
+            searcher.close();
+            assert inMemoryReader.assertMemorySegmentStatus(false);
+            final TranslogLeafReader translogLeafReader = new TranslogLeafReader(index);
+            return new GetResult(new Engine.Searcher("realtime_get", translogLeafReader,
+                IndexSearcher.getDefaultSimilarity(), null, IndexSearcher.getDefaultQueryCachingPolicy(), translogLeafReader),
+                new VersionsAndSeqNoResolver.DocIdAndVersion(
+                    0, index.version(), index.seqNo(), index.primaryTerm(), translogLeafReader, 0), true);
+        } else {
+            assert inMemoryReader.assertMemorySegmentStatus(true);
+            return getFromSearcher(get, wrappedSearcher);
+        }
+    }
+
     @Override
-    public GetResult get(Get get, BiFunction<String, SearcherScope, Engine.Searcher> searcherFactory) throws EngineException {
+    public GetResult get(Get get, DocumentMapper mapper, Function<Engine.Searcher, Engine.Searcher> searcherWrapper) {
         assert Objects.equals(get.uid().field(), IdFieldMapper.NAME) : get.uid().field();
         try (ReleasableLock ignored = readLock.acquire()) {
             ensureOpen();
-            SearcherScope scope;
             if (get.realtime()) {
-                VersionValue versionValue = null;
+                final VersionValue versionValue;
                 try (Releasable ignore = versionMap.acquireLock(get.uid().bytes())) {
                     // we need to lock here to access the version map to do this truly in RT
                     versionValue = getVersionFromMap(get.uid().bytes());
@@ -649,15 +670,9 @@ public GetResult get(Get get, BiFunction<String, SearcherScope, Engine.Searcher>
                         // the update call doesn't need the consistency since it's source only + _parent but parent can go away in 7.0
                         if (versionValue.getLocation() != null) {
                             try {
-                                Translog.Operation operation = translog.readOperation(versionValue.getLocation());
+                                final Translog.Operation operation = translog.readOperation(versionValue.getLocation());
                                 if (operation != null) {
-                                    // in the case of a already pruned translog generation we might get null here - yet very unlikely
-                                    final Translog.Index index = (Translog.Index) operation;
-                                    TranslogLeafReader reader = new TranslogLeafReader(index);
-                                    return new GetResult(new Engine.Searcher("realtime_get", reader,
-                                        IndexSearcher.getDefaultSimilarity(), null, IndexSearcher.getDefaultQueryCachingPolicy(), reader),
-                                        new VersionsAndSeqNoResolver.DocIdAndVersion(0, index.version(), index.seqNo(), index.primaryTerm(),
-                                            reader, 0), true);
+                                    return getFromTranslog(get, (Translog.Index) operation, mapper, searcherWrapper);
                                 }
                             } catch (IOException e) {
                                 maybeFailEngine("realtime_get", e); // lets check if the translog has failed with a tragic event
@@ -670,14 +685,11 @@ public GetResult get(Get get, BiFunction<String, SearcherScope, Engine.Searcher>
                     assert versionValue.seqNo >= 0 : versionValue;
                     refreshIfNeeded("realtime_get", versionValue.seqNo);
                 }
-                scope = SearcherScope.INTERNAL;
+                return getFromSearcher(get, acquireSearcher("realtime_get", SearcherScope.INTERNAL, searcherWrapper));
             } else {
                 // we expose what has been externally expose in a point in time snapshot via an explicit refresh
-                scope = SearcherScope.EXTERNAL;
+                return getFromSearcher(get, acquireSearcher("get", SearcherScope.EXTERNAL, searcherWrapper));
             }
-
-            // no version, get the version from the index, we know that we refresh on flush
-            return getFromSearcher(get, searcherFactory, scope);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
index 4bc697e018ee4..c9d9baaef0d26 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.util.concurrent.ReleasableLock;
 import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
@@ -48,7 +49,6 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
-import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.stream.Stream;
 
@@ -219,8 +219,8 @@ private static TranslogStats translogStats(final EngineConfig config, final Segm
     }
 
     @Override
-    public GetResult get(Get get, BiFunction<String, SearcherScope, Engine.Searcher> searcherFactory) throws EngineException {
-        return getFromSearcher(get, searcherFactory, SearcherScope.EXTERNAL);
+    public GetResult get(Get get, DocumentMapper mapper, Function<Searcher, Searcher> searcherWrapper) {
+        return getFromSearcher(get, acquireSearcher("get", SearcherScope.EXTERNAL, searcherWrapper));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java
new file mode 100644
index 0000000000000..92dfbb826e088
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.engine;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafMetaData;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.translog.Translog;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * A {@link DirectoryReader} contains a single leaf reader delegating to an in-memory Lucene segment that is lazily created from
+ * a single document.
+ */
+final class SingleDocDirectoryReader extends DirectoryReader {
+    private final SingleDocLeafReader leafReader;
+
+    SingleDocDirectoryReader(ShardId shardId, Translog.Index operation, DocumentMapper mapper, Analyzer analyzer) throws IOException {
+        this(new SingleDocLeafReader(shardId, operation, mapper, analyzer));
+    }
+
+    private SingleDocDirectoryReader(SingleDocLeafReader leafReader) throws IOException {
+        super(leafReader.directory, new LeafReader[]{leafReader});
+        this.leafReader = leafReader;
+    }
+
+    boolean assertMemorySegmentStatus(boolean loaded) {
+        return leafReader.assertMemorySegmentStatus(loaded);
+    }
+
+    private static UnsupportedOperationException unsupported() {
+        assert false : "unsupported operation";
+        return new UnsupportedOperationException();
+    }
+
+    @Override
+    protected DirectoryReader doOpenIfChanged() {
+        throw unsupported();
+    }
+
+    @Override
+    protected DirectoryReader doOpenIfChanged(IndexCommit commit) {
+        throw unsupported();
+    }
+
+    @Override
+    protected DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) {
+        throw unsupported();
+    }
+
+    @Override
+    public long getVersion() {
+        throw unsupported();
+    }
+
+    @Override
+    public boolean isCurrent() {
+        throw unsupported();
+    }
+
+    @Override
+    public IndexCommit getIndexCommit() {
+        throw unsupported();
+    }
+
+    @Override
+    protected void doClose() throws IOException {
+        leafReader.close();
+    }
+
+    @Override
+    public CacheHelper getReaderCacheHelper() {
+        return leafReader.getReaderCacheHelper();
+    }
+
+    private static class SingleDocLeafReader extends LeafReader {
+
+        private final ShardId shardId;
+        private final Translog.Index operation;
+        private final DocumentMapper mapper;
+        private final Analyzer analyzer;
+        private final Directory directory;
+        private final AtomicReference<LeafReader> delegate = new AtomicReference<>();
+
+        SingleDocLeafReader(ShardId shardId, Translog.Index operation, DocumentMapper mapper, Analyzer analyzer) {
+            this.shardId = shardId;
+            this.operation = operation;
+            this.mapper = mapper;
+            this.analyzer = analyzer;
+            this.directory = new ByteBuffersDirectory();
+        }
+
+        private LeafReader getDelegate() {
+            ensureOpen();
+            LeafReader reader = delegate.get();
+            if (reader == null) {
+                synchronized (this) {
+                    reader = delegate.get();
+                    if (reader == null) {
+                        reader = createInMemoryLeafReader();
+                        final LeafReader existing = delegate.getAndSet(reader);
+                        assert existing == null;
+                    }
+                }
+            }
+            return reader;
+        }
+
+        private LeafReader createInMemoryLeafReader() {
+            assert Thread.holdsLock(this);
+            final ParsedDocument parsedDocs = mapper.parse(new SourceToParse(shardId.getIndexName(), operation.id(),
+                operation.source(), XContentHelper.xContentType(operation.source()), operation.routing()));
+            parsedDocs.updateSeqID(operation.seqNo(), operation.primaryTerm());
+            parsedDocs.version().setLongValue(operation.version());
+            final IndexWriterConfig writeConfig = new IndexWriterConfig(analyzer).setOpenMode(IndexWriterConfig.OpenMode.CREATE);
+            try (IndexWriter writer = new IndexWriter(directory, writeConfig)) {
+                writer.addDocument(parsedDocs.rootDoc());
+                final DirectoryReader reader = open(writer);
+                if (reader.leaves().size() != 1 || reader.leaves().get(0).reader().numDocs() != 1) {
+                    reader.close();
+                    throw new IllegalStateException("Expected a single document segment; " +
+                        "but [" + reader.leaves().size() + " segments with " + reader.leaves().get(0).reader().numDocs() + " documents");
+                }
+                return reader.leaves().get(0).reader();
+            } catch (IOException e) {
+                throw new EngineException(shardId, "failed to create an in-memory segment for get [" + operation.id() + "]", e);
+            }
+        }
+
+        @Override
+        public CacheHelper getCoreCacheHelper() {
+            return getDelegate().getCoreCacheHelper();
+        }
+
+        @Override
+        public CacheHelper getReaderCacheHelper() {
+            return getDelegate().getReaderCacheHelper();
+        }
+
+        @Override
+        public Terms terms(String field) throws IOException {
+            return getDelegate().terms(field);
+        }
+
+        @Override
+        public NumericDocValues getNumericDocValues(String field) throws IOException {
+            return getDelegate().getNumericDocValues(field);
+        }
+
+        @Override
+        public BinaryDocValues getBinaryDocValues(String field) throws IOException {
+            return getDelegate().getBinaryDocValues(field);
+        }
+
+        @Override
+        public SortedDocValues getSortedDocValues(String field) throws IOException {
+            return getDelegate().getSortedDocValues(field);
+        }
+
+        @Override
+        public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
+            return getDelegate().getSortedNumericDocValues(field);
+        }
+
+        @Override
+        public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
+            return getDelegate().getSortedSetDocValues(field);
+        }
+
+        @Override
+        public NumericDocValues getNormValues(String field) throws IOException {
+            return getDelegate().getNormValues(field);
+        }
+
+        @Override
+        public FieldInfos getFieldInfos() {
+            return getDelegate().getFieldInfos();
+        }
+
+        @Override
+        public Bits getLiveDocs() {
+            return getDelegate().getLiveDocs();
+        }
+
+        @Override
+        public PointValues getPointValues(String field) throws IOException {
+            return getDelegate().getPointValues(field);
+        }
+
+        @Override
+        public void checkIntegrity() throws IOException {
+        }
+
+        @Override
+        public LeafMetaData getMetaData() {
+            return getDelegate().getMetaData();
+        }
+
+        @Override
+        public Fields getTermVectors(int docID) throws IOException {
+            return getDelegate().getTermVectors(docID);
+        }
+
+        @Override
+        public int numDocs() {
+            return 1;
+        }
+
+        @Override
+        public int maxDoc() {
+            return 1;
+        }
+
+        synchronized boolean assertMemorySegmentStatus(boolean loaded) {
+            if (loaded) {
+                assert delegate.get() != null :
+                    "Expected an in memory segment was loaded; but it wasn't. Please check the reader wrapper implementation";
+            } else {
+                assert delegate.get() == null :
+                    "Expected an in memory segment wasn't loaded; but it was. Please check the reader wrapper implementation";
+            }
+            return true;
+        }
+
+        @Override
+        public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+            assert assertMemorySegmentStatus(true);
+            getDelegate().document(docID, visitor);
+        }
+
+        @Override
+        protected void doClose() throws IOException {
+            IOUtils.close(delegate.get(), directory);
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index d2e469012cca3..498c15f243868 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -923,7 +923,7 @@ public Engine.GetResult get(Engine.Get get) {
         if (mapper == null) {
             return GetResult.NOT_EXISTS;
         }
-        return getEngine().get(get, this::acquireSearcher);
+        return getEngine().get(get, mapper, this::wrapSearcher);
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 7f18327f929f2..129bde0b45d76 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -55,6 +55,7 @@
 import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortedSetSortField;
@@ -108,6 +109,7 @@
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
@@ -123,6 +125,7 @@
 import org.elasticsearch.index.seqno.RetentionLeases;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.shard.SearcherHelper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
 import org.elasticsearch.index.store.Store;
@@ -695,8 +698,7 @@ public void testConcurrentGetAndFlush() throws Exception {
         engine.index(indexForDoc(doc));
 
         final AtomicReference<Engine.GetResult> latestGetResult = new AtomicReference<>();
-        final BiFunction<String, Engine.SearcherScope, Engine.Searcher> searcherFactory = engine::acquireSearcher;
-        latestGetResult.set(engine.get(newGet(true, doc), searcherFactory));
+        latestGetResult.set(engine.get(newGet(true, doc), docMapper(), randomSearcherWrapper()));
         final AtomicBoolean flushFinished = new AtomicBoolean(false);
         final CyclicBarrier barrier = new CyclicBarrier(2);
         Thread getThread = new Thread(() -> {
@@ -710,7 +712,7 @@ public void testConcurrentGetAndFlush() throws Exception {
                 if (previousGetResult != null) {
                     previousGetResult.close();
                 }
-                latestGetResult.set(engine.get(newGet(true, doc), searcherFactory));
+                latestGetResult.set(engine.get(newGet(true, doc), docMapper(), randomSearcherWrapper()));
                 if (latestGetResult.get().exists() == false) {
                     break;
                 }
@@ -726,6 +728,7 @@ public void testConcurrentGetAndFlush() throws Exception {
     }
 
     public void testSimpleOperations() throws Exception {
+        final DocumentMapper mapper = docMapper();
         engine.refresh("warm_up");
         Engine.Searcher searchResult = engine.acquireSearcher("test");
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
@@ -747,18 +750,18 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // but, not there non realtime
-        try (Engine.GetResult getResult = engine.get(newGet(false, doc), searcherFactory)) {
+        try (Engine.GetResult getResult = engine.get(newGet(false, doc), mapper, randomSearcherWrapper())) {
             assertThat(getResult.exists(), equalTo(false));
         }
 
         // but, we can still get it (in realtime)
-        try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) {
+        try (Engine.GetResult getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper())) {
             assertThat(getResult.exists(), equalTo(true));
             assertThat(getResult.docIdAndVersion(), notNullValue());
         }
 
         // but not real time is not yet visible
-        try (Engine.GetResult getResult = engine.get(newGet(false, doc), searcherFactory)) {
+        try (Engine.GetResult getResult = engine.get(newGet(false, doc), mapper, randomSearcherWrapper())) {
             assertThat(getResult.exists(), equalTo(false));
         }
 
@@ -773,7 +776,7 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // also in non realtime
-        try (Engine.GetResult getResult = engine.get(newGet(false, doc), searcherFactory)) {
+        try (Engine.GetResult getResult = engine.get(newGet(false, doc), mapper, randomSearcherWrapper())) {
             assertThat(getResult.exists(), equalTo(true));
             assertThat(getResult.docIdAndVersion(), notNullValue());
         }
@@ -781,8 +784,8 @@ public void testSimpleOperations() throws Exception {
         // now do an update
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
-        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", null, document, B_2, null);
+        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(SOURCE), SourceFieldMapper.Defaults.FIELD_TYPE));
+        doc = testParsedDocument("1", null, document, SOURCE, null);
         engine.index(indexForDoc(doc));
 
         // its not updated yet...
@@ -795,7 +798,7 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // but, we can still get it (in realtime)
-        try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) {
+        try (Engine.GetResult getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper())) {
             assertThat(getResult.exists(), equalTo(true));
             assertThat(getResult.docIdAndVersion(), notNullValue());
         }
@@ -824,7 +827,7 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
 
         // but, get should not see it (in realtime)
-        try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) {
+        try (Engine.GetResult getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper())) {
             assertThat(getResult.exists(), equalTo(false));
         }
 
@@ -870,7 +873,7 @@ public void testSimpleOperations() throws Exception {
         engine.flush();
 
         // and, verify get (in real time)
-        try (Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory)) {
+        try (Engine.GetResult getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper())) {
             assertThat(getResult.exists(), equalTo(true));
             assertThat(getResult.docIdAndVersion(), notNullValue());
         }
@@ -903,6 +906,53 @@ public void testSimpleOperations() throws Exception {
         searchResult.close();
     }
 
+    public void testGetWithSearcherWrapper() throws Exception {
+        engine.refresh("warm_up");
+        engine.index(indexForDoc(createParsedDoc("1", null)));
+        assertThat(engine.lastRefreshedCheckpoint(), equalTo(NO_OPS_PERFORMED));
+        try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(), randomSearcherWrapper())) {
+            // we do not track the translog location yet
+            assertTrue(get.exists());
+            assertFalse(get.isFromTranslog());
+        }
+        // refresh triggered, as we did not track translog location until the first realtime get.
+        assertThat(engine.lastRefreshedCheckpoint(), equalTo(0L));
+
+        engine.index(indexForDoc(createParsedDoc("1", null)));
+        try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(), searcher -> searcher)) {
+            assertTrue(get.exists());
+            assertTrue(get.isFromTranslog());
+        }
+        assertThat(engine.lastRefreshedCheckpoint(), equalTo(0L)); // no refresh; just read from translog
+        if (randomBoolean()) {
+            engine.index(indexForDoc(createParsedDoc("1", null)));
+        }
+        try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(),
+            searcher -> SearcherHelper.wrapSearcher(searcher, reader -> new MatchingDirectoryReader(reader, new MatchAllDocsQuery())))) {
+            assertTrue(get.exists());
+            assertFalse(get.isFromTranslog());
+        }
+
+        try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(),
+            searcher -> SearcherHelper.wrapSearcher(searcher, reader -> new MatchingDirectoryReader(reader, new MatchNoDocsQuery())))) {
+            assertFalse(get.exists());
+            assertFalse(get.isFromTranslog());
+        }
+
+        try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(),
+            searcher -> SearcherHelper.wrapSearcher(searcher, reader -> new MatchingDirectoryReader(reader, new TermQuery(newUid("1")))))) {
+            assertTrue(get.exists());
+            assertFalse(get.isFromTranslog());
+        }
+
+        try (Engine.GetResult get = engine.get(new Engine.Get(true, true, "1"), docMapper(),
+            searcher -> SearcherHelper.wrapSearcher(searcher, reader -> new MatchingDirectoryReader(reader, new TermQuery(newUid("2")))))) {
+            assertFalse(get.exists());
+            assertFalse(get.isFromTranslog());
+        }
+        assertThat("no refresh, just read from translog or in-memory segment", engine.lastRefreshedCheckpoint(), equalTo(0L));
+    }
+
     public void testSearchResultRelease() throws Exception {
         engine.refresh("warm_up");
         Engine.Searcher searchResult = engine.acquireSearcher("test");
@@ -1156,7 +1206,7 @@ public void testVersionedUpdate() throws IOException {
         Engine.Index create = new Engine.Index(newUid(doc), primaryTerm.get(), doc, Versions.MATCH_DELETED);
         Engine.IndexResult indexResult = engine.index(create);
         assertThat(indexResult.getVersion(), equalTo(1L));
-        try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), searcherFactory)) {
+        try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), docMapper(), randomSearcherWrapper())) {
             assertEquals(1, get.version());
         }
 
@@ -1164,7 +1214,7 @@ public void testVersionedUpdate() throws IOException {
         Engine.IndexResult update_1_result = engine.index(update_1);
         assertThat(update_1_result.getVersion(), equalTo(2L));
 
-        try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), searcherFactory)) {
+        try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), docMapper(), randomSearcherWrapper())) {
             assertEquals(2, get.version());
         }
 
@@ -1172,14 +1222,13 @@ public void testVersionedUpdate() throws IOException {
         Engine.IndexResult update_2_result = engine.index(update_2);
         assertThat(update_2_result.getVersion(), equalTo(3L));
 
-        try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), searcherFactory)) {
+        try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), docMapper(), randomSearcherWrapper())) {
             assertEquals(3, get.version());
         }
 
     }
 
     public void testGetIfSeqNoIfPrimaryTerm() throws IOException {
-        final BiFunction<String, Engine.SearcherScope, Engine.Searcher> searcherFactory = engine::acquireSearcher;
 
         ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);
         Engine.Index create = new Engine.Index(newUid(doc), primaryTerm.get(), doc, Versions.MATCH_DELETED);
@@ -1193,22 +1242,22 @@ public void testGetIfSeqNoIfPrimaryTerm() throws IOException {
         try (Engine.GetResult get = engine.get(
             new Engine.Get(true, true, doc.id())
                 .setIfSeqNo(indexResult.getSeqNo()).setIfPrimaryTerm(primaryTerm.get()),
-            searcherFactory)) {
+            docMapper(), randomSearcherWrapper())) {
             assertEquals(indexResult.getSeqNo(), get.docIdAndVersion().seqNo);
         }
 
         expectThrows(VersionConflictEngineException.class, () -> engine.get(new Engine.Get(true, false, doc.id())
                 .setIfSeqNo(indexResult.getSeqNo() + 1).setIfPrimaryTerm(primaryTerm.get()),
-            searcherFactory));
+            docMapper(), randomSearcherWrapper()));
 
         expectThrows(VersionConflictEngineException.class, () -> engine.get(new Engine.Get(true, false, doc.id())
                 .setIfSeqNo(indexResult.getSeqNo()).setIfPrimaryTerm(primaryTerm.get() + 1),
-            searcherFactory));
+            docMapper(), randomSearcherWrapper()));
 
         final VersionConflictEngineException versionConflictEngineException
             = expectThrows(VersionConflictEngineException.class, () -> engine.get(new Engine.Get(true, false, doc.id())
                 .setIfSeqNo(indexResult.getSeqNo() + 1).setIfPrimaryTerm(primaryTerm.get() + 1),
-            searcherFactory));
+            docMapper(), randomSearcherWrapper()));
         assertThat(versionConflictEngineException.getStackTrace(), emptyArray());
     }
 
@@ -1954,7 +2003,6 @@ class OpAndVersion {
         ParsedDocument doc = testParsedDocument("1", null, testDocument(), bytesArray(""), null);
         final Term uidTerm = newUid(doc);
         engine.index(indexForDoc(doc));
-        final BiFunction<String, Engine.SearcherScope, Engine.Searcher> searcherFactory = engine::acquireSearcher;
         for (int i = 0; i < thread.length; i++) {
             thread[i] = new Thread(() -> {
                 startGun.countDown();
@@ -1964,7 +2012,7 @@ class OpAndVersion {
                     throw new AssertionError(e);
                 }
                 for (int op = 0; op < opsPerThread; op++) {
-                    try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), searcherFactory)) {
+                    try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), docMapper(), randomSearcherWrapper())) {
                         FieldsVisitor visitor = new FieldsVisitor(true);
                         get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
                         List<String> values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString()));
@@ -2006,7 +2054,7 @@ class OpAndVersion {
             assertTrue(op.added + " should not exist", exists);
         }
 
-        try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), searcherFactory)) {
+        try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id()), docMapper(), randomSearcherWrapper())) {
             FieldsVisitor visitor = new FieldsVisitor(true);
             get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
             List<String> values = Arrays.asList(Strings.commaDelimitedListToStringArray(visitor.source().utf8ToString()));
@@ -2402,7 +2450,7 @@ public void testEnableGcDeletes() throws Exception {
              Engine engine = createEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(), null))) {
             engine.config().setEnableGcDeletes(false);
 
-            final BiFunction<String, Engine.SearcherScope, Engine.Searcher> searcherFactory = engine::acquireSearcher;
+            final DocumentMapper mapper = docMapper();
 
             // Add document
             Document document = testDocument();
@@ -2418,7 +2466,7 @@ public void testEnableGcDeletes() throws Exception {
                 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0));
 
             // Get should not find the document
-            Engine.GetResult getResult = engine.get(newGet(true, doc), searcherFactory);
+            Engine.GetResult getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper());
             assertThat(getResult.exists(), equalTo(false));
 
             // Give the gc pruning logic a chance to kick in
@@ -2433,7 +2481,7 @@ public void testEnableGcDeletes() throws Exception {
                 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0));
 
             // Get should not find the document (we never indexed uid=2):
-            getResult = engine.get(new Engine.Get(true, false, "2"), searcherFactory);
+            getResult = engine.get(new Engine.Get(true, false, "2"), mapper, randomSearcherWrapper());
             assertThat(getResult.exists(), equalTo(false));
 
             // Try to index uid=1 with a too-old version, should fail:
@@ -2444,7 +2492,7 @@ public void testEnableGcDeletes() throws Exception {
             assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
 
             // Get should still not find the document
-            getResult = engine.get(newGet(true, doc), searcherFactory);
+            getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper());
             assertThat(getResult.exists(), equalTo(false));
 
             // Try to index uid=2 with a too-old version, should fail:
@@ -2455,7 +2503,7 @@ public void testEnableGcDeletes() throws Exception {
             assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
 
             // Get should not find the document
-            getResult = engine.get(newGet(true, doc), searcherFactory);
+            getResult = engine.get(newGet(true, doc), mapper, randomSearcherWrapper());
             assertThat(getResult.exists(), equalTo(false));
         }
     }
@@ -3968,7 +4016,7 @@ public void testOutOfOrderSequenceNumbersWithVersionConflict() throws IOExceptio
         }
 
         assertThat(engine.getProcessedLocalCheckpoint(), equalTo(expectedLocalCheckpoint));
-        try (Engine.GetResult result = engine.get(new Engine.Get(true, false, "1"), searcherFactory)) {
+        try (Engine.GetResult result = engine.get(new Engine.Get(true, false, "1"), docMapper(), randomSearcherWrapper())) {
             assertThat(result.exists(), equalTo(exists));
         }
     }
@@ -4851,7 +4899,7 @@ public void testStressShouldPeriodicallyFlush() throws Exception {
     }
 
     public void testStressUpdateSameDocWhileGettingIt() throws IOException, InterruptedException {
-        final int iters = randomIntBetween(1, 15);
+        final int iters = randomIntBetween(1, 1);
         for (int i = 0; i < iters; i++) {
             // this is a reproduction of https://github.com/elastic/elasticsearch/issues/28714
             try (Store store = createStore(); InternalEngine engine = createEngine(store, createTempDir())) {
@@ -4893,13 +4941,15 @@ public void testStressUpdateSameDocWhileGettingIt() throws IOException, Interrup
                 CountDownLatch awaitStarted = new CountDownLatch(1);
                 Thread thread = new Thread(() -> {
                     awaitStarted.countDown();
-                    try (Engine.GetResult getResult = engine.get(new Engine.Get(true, false, doc3.id()), engine::acquireSearcher)) {
+                    try (Engine.GetResult getResult = engine.get(
+                        new Engine.Get(true, false, doc3.id()), docMapper(), searcher -> searcher)) {
                         assertTrue(getResult.exists());
                     }
                 });
                 thread.start();
                 awaitStarted.await();
-                try (Engine.GetResult getResult = engine.get(new Engine.Get(true, false, doc.id()), engine::acquireSearcher)) {
+                try (Engine.GetResult getResult = engine.get(new Engine.Get(true, false, doc.id()), docMapper(),
+                    searcher -> SearcherHelper.wrapSearcher(searcher, r -> new MatchingDirectoryReader(r, new MatchAllDocsQuery())))) {
                     assertFalse(getResult.exists());
                 }
                 thread.join();
@@ -5848,7 +5898,7 @@ public void afterRefresh(boolean didRefresh) {
                         int iters = randomIntBetween(1, 10);
                         for (int i = 0; i < iters; i++) {
                             ParsedDocument doc = createParsedDoc(randomFrom(ids), null);
-                            try (Engine.GetResult getResult = engine.get(newGet(true, doc), engine::acquireSearcher)) {
+                            try (Engine.GetResult getResult = engine.get(newGet(true, doc), docMapper(), randomSearcherWrapper())) {
                                 assertThat(getResult.exists(), equalTo(true));
                                 assertThat(getResult.docIdAndVersion(), notNullValue());
                             }
diff --git a/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java
index 563bd0acc2751..a9731539fc446 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java
@@ -100,7 +100,7 @@ public void testReadOnlyEngine() throws Exception {
                 assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
                 assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
                 assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
-                try (Engine.GetResult getResult = readOnlyEngine.get(get, readOnlyEngine::acquireSearcher)) {
+                try (Engine.GetResult getResult = readOnlyEngine.get(get, docMapper(), randomSearcherWrapper())) {
                     assertTrue(getResult.exists());
                 }
             }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index b6e72863279c0..ce108017d6028 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -47,6 +47,7 @@
 import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.engine.EngineTestCase;
 import org.elasticsearch.index.engine.InternalEngine;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -344,7 +345,8 @@ public void testLotsOfThreads() throws Exception {
                         listener.assertNoError();
 
                         Engine.Get get = new Engine.Get(false, false, threadId);
-                        try (Engine.GetResult getResult = engine.get(get, engine::acquireSearcher)) {
+                        final DocumentMapper mapper = EngineTestCase.docMapper();
+                        try (Engine.GetResult getResult = engine.get(get, mapper, EngineTestCase.randomSearcherWrapper())) {
                             assertTrue("document not found", getResult.exists());
                             assertEquals(iteration, getResult.version());
                             org.apache.lucene.document.Document document =
diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java
index d3bde9d174b1a..44a316242b98f 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.EngineTestCase;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
@@ -117,7 +118,7 @@ private void runGetFromTranslogWithOptions(String docToIndex, String sourceOptio
                 "\"bar\":  { \"type\": " + fieldType + "}}, \"_source\": { " + sourceOptions + "}}}")
             .settings(settings)
             .primaryTerm(0, 1).build();
-        IndexShard primary = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, null);
+        IndexShard primary = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, EngineTestCase.randomReaderWrapper());
         recoverShardFromStore(primary);
         Engine.IndexResult test = indexDoc(primary, "test", "0", docToIndex);
         assertTrue(primary.getEngine().refreshNeeded());
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
index aa9c0c42c1eb2..ca2f9a36b0044 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
@@ -28,6 +28,8 @@
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FilterLeafReader;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -40,14 +42,19 @@
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ReferenceManager;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TotalHitCountCollector;
+import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
@@ -55,6 +62,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.common.CheckedBiFunction;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
@@ -94,6 +102,7 @@
 import org.elasticsearch.index.seqno.ReplicationTracker;
 import org.elasticsearch.index.seqno.RetentionLeases;
 import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.shard.SearcherHelper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
@@ -110,6 +119,7 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.charset.Charset;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -1184,6 +1194,14 @@ public static MapperService createMapperService() throws IOException {
         return mapperService;
     }
 
+    public static DocumentMapper docMapper() {
+        try {
+            return createMapperService().documentMapper();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
     /**
      * Exposes a translog associated with the given engine for testing purpose.
      */
@@ -1259,4 +1277,78 @@ public static long getInFlightDocCount(Engine engine) {
     public static void assertNoInFlightDocuments(Engine engine) throws Exception {
         assertBusy(() -> assertThat(getInFlightDocCount(engine), equalTo(0L)));
     }
+
+    public static final class MatchingDirectoryReader extends FilterDirectoryReader {
+        private final Query query;
+
+        public MatchingDirectoryReader(DirectoryReader in, Query query) throws IOException {
+            super(in, new SubReaderWrapper() {
+                @Override
+                public LeafReader wrap(LeafReader leaf) {
+                    try {
+                        final IndexSearcher searcher = new IndexSearcher(leaf);
+                        final Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE_NO_SCORES, 1.0f);
+                        final Scorer scorer = weight.scorer(leaf.getContext());
+                        final DocIdSetIterator iterator = scorer != null ? scorer.iterator() : null;
+                        final FixedBitSet liveDocs = new FixedBitSet(leaf.maxDoc());
+                        if (iterator != null) {
+                            for (int docId = iterator.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = iterator.nextDoc()) {
+                                if (leaf.getLiveDocs() == null || leaf.getLiveDocs().get(docId)) {
+                                    liveDocs.set(docId);
+                                }
+                            }
+                        }
+                        return new FilterLeafReader(leaf) {
+                            @Override
+                            public Bits getLiveDocs() {
+                                return liveDocs;
+                            }
+
+                            @Override
+                            public CacheHelper getCoreCacheHelper() {
+                                return leaf.getCoreCacheHelper();
+                            }
+
+                            @Override
+                            public CacheHelper getReaderCacheHelper() {
+                                return null; // modify liveDocs
+                            }
+                        };
+                    } catch (IOException e) {
+                        throw new UncheckedIOException(e);
+                    }
+                }
+            });
+            this.query = query;
+        }
+
+        @Override
+        protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
+            return new MatchingDirectoryReader(in, query);
+        }
+
+        @Override
+        public CacheHelper getReaderCacheHelper() {
+            // TODO: We should not return the ReaderCacheHelper if we modify the liveDocs,
+            // but some caching components (e.g., global ordinals) require this cache key.
+            return in.getReaderCacheHelper();
+        }
+    }
+
+    public static CheckedFunction<DirectoryReader, DirectoryReader, IOException> randomReaderWrapper() {
+        if (randomBoolean()) {
+            return reader -> reader;
+        } else {
+            return reader -> new MatchingDirectoryReader(reader, new MatchAllDocsQuery());
+        }
+    }
+
+    public static Function<Engine.Searcher, Engine.Searcher> randomSearcherWrapper() {
+        if (randomBoolean()) {
+            return Function.identity();
+        } else {
+            final CheckedFunction<DirectoryReader, DirectoryReader, IOException> readerWrapper = randomReaderWrapper();
+            return searcher -> SearcherHelper.wrapSearcher(searcher, readerWrapper);
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index c9fe7ac9a3199..d677891743cfa 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -367,6 +367,9 @@ protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMe
             storeProvider = is -> createStore(is, shardPath);
         }
         final Store store = storeProvider.apply(indexSettings);
+        if (indexReaderWrapper == null && randomBoolean()) {
+            indexReaderWrapper = EngineTestCase.randomReaderWrapper();
+        }
         boolean success = false;
         try {
             IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/SearcherHelper.java b/test/framework/src/main/java/org/elasticsearch/index/shard/SearcherHelper.java
new file mode 100644
index 0000000000000..dd5a769c8d999
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/SearcherHelper.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.shard;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.index.engine.Engine;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+public class SearcherHelper {
+
+    public static Engine.Searcher wrapSearcher(Engine.Searcher engineSearcher,
+                                               CheckedFunction<DirectoryReader, DirectoryReader, IOException> readerWrapper) {
+        try {
+            return IndexShard.wrapSearcher(engineSearcher, readerWrapper);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+}

From 191f36d5ec8c4b8e90d378d4e307d754f8652319 Mon Sep 17 00:00:00 2001
From: Ignacio Vera <iverase@gmail.com>
Date: Tue, 10 Nov 2020 07:21:00 +0100
Subject: [PATCH 34/34] Repackage some classes in the Spatial plugin (#64758)

---
 .../java/org/elasticsearch/xpack/spatial/SpatialPlugin.java   | 2 +-
 .../{ => plain}/AbstractAtomicGeoShapeShapeFieldData.java     | 4 +++-
 .../{ => plain}/AbstractLatLonShapeIndexFieldData.java        | 4 +++-
 .../{ => plain}/LatLonShapeDVAtomicShapeFieldData.java        | 4 +++-
 .../index/mapper/GeoShapeWithDocValuesFieldMapper.java        | 2 +-
 .../aggregations/metrics/GeoShapeCentroidAggregator.java      | 2 +-
 .../aggregations/metrics/GeoShapeCentroidAggregatorTests.java | 2 +-
 7 files changed, 13 insertions(+), 7 deletions(-)
 rename x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/{ => plain}/AbstractAtomicGeoShapeShapeFieldData.java (87%)
 rename x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/{ => plain}/AbstractLatLonShapeIndexFieldData.java (95%)
 rename x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/{ => plain}/LatLonShapeDVAtomicShapeFieldData.java (92%)
 rename x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/{ => search}/aggregations/metrics/GeoShapeCentroidAggregator.java (99%)
 rename x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/{ => search}/aggregations/metrics/GeoShapeCentroidAggregatorTests.java (99%)

diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java
index 940afdc458400..1f5564eab3991 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.xpack.spatial.action.SpatialInfoTransportAction;
 import org.elasticsearch.xpack.spatial.action.SpatialStatsTransportAction;
 import org.elasticsearch.xpack.spatial.action.SpatialUsageTransportAction;
-import org.elasticsearch.xpack.spatial.aggregations.metrics.GeoShapeCentroidAggregator;
+import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeCentroidAggregator;
 import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper;
 import org.elasticsearch.xpack.spatial.index.mapper.PointFieldMapper;
 import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper;
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractAtomicGeoShapeShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java
similarity index 87%
rename from x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractAtomicGeoShapeShapeFieldData.java
rename to x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java
index 32185760bd84c..262b03c2c7d0a 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractAtomicGeoShapeShapeFieldData.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java
@@ -4,11 +4,13 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.spatial.index.fielddata;
+package org.elasticsearch.xpack.spatial.index.fielddata.plain;
 
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues;
+import org.elasticsearch.xpack.spatial.index.fielddata.LeafGeoShapeFieldData;
 
 import java.util.Collection;
 import java.util.Collections;
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractLatLonShapeIndexFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractLatLonShapeIndexFieldData.java
similarity index 95%
rename from x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractLatLonShapeIndexFieldData.java
rename to x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractLatLonShapeIndexFieldData.java
index 0dc41abfdf8de..df642588ca51c 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/AbstractLatLonShapeIndexFieldData.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractLatLonShapeIndexFieldData.java
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.spatial.index.fielddata;
+package org.elasticsearch.xpack.spatial.index.fielddata.plain;
 
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
@@ -21,6 +21,8 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.sort.BucketedSort;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.xpack.spatial.index.fielddata.IndexGeoShapeFieldData;
+import org.elasticsearch.xpack.spatial.index.fielddata.LeafGeoShapeFieldData;
 
 public abstract class AbstractLatLonShapeIndexFieldData implements IndexGeoShapeFieldData {
     protected final String fieldName;
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LatLonShapeDVAtomicShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java
similarity index 92%
rename from x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LatLonShapeDVAtomicShapeFieldData.java
rename to x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java
index a1eed2faa24b8..97820a2025870 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LatLonShapeDVAtomicShapeFieldData.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.spatial.index.fielddata;
+package org.elasticsearch.xpack.spatial.index.fielddata.plain;
 
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocValues;
@@ -12,6 +12,8 @@
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeometryDocValueReader;
 import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java
index 1f7627968a3e2..5199b9ba9482c 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.xpack.spatial.index.fielddata.AbstractLatLonShapeIndexFieldData;
+import org.elasticsearch.xpack.spatial.index.fielddata.plain.AbstractLatLonShapeIndexFieldData;
 import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType;
 
 import java.util.Arrays;
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregator.java
similarity index 99%
rename from x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregator.java
rename to x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregator.java
index 8455fa4153f4c..8c6783c2b2394 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregator.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregator.java
@@ -5,7 +5,7 @@
  */
 
 
-package org.elasticsearch.xpack.spatial.aggregations.metrics;
+package org.elasticsearch.xpack.spatial.search.aggregations.metrics;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.geo.GeoPoint;
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java
similarity index 99%
rename from x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregatorTests.java
rename to x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java
index 132af642a62e7..eda3cf5ab4a2b 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregatorTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.spatial.aggregations.metrics;
+package org.elasticsearch.xpack.spatial.search.aggregations.metrics;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.LatLonDocValuesField;