diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
index 544cc47d4a9c0..f78a98f8dc827 100644
--- a/build-tools-internal/version.properties
+++ b/build-tools-internal/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 9.0.0-snapshot-cfd9f9f98f7
+lucene            = 9.0.0-snapshot-8b68bf60c98
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17+35
diff --git a/build.gradle b/build.gradle
index 60a2735f8cbe2..1569cec42e9a5 100644
--- a/build.gradle
+++ b/build.gradle
@@ -132,9 +132,9 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
+boolean bwc_tests_enabled = true
 // place a PR link here when committing bwc changes:
-String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/79385"
+String bwc_tests_disabled_issue = ""
 /*
  * FIPS 140-2 behavior was fixed in 7.11.0. Before that there is no way to run elasticsearch in a
  * JVM that is properly configured to be in fips mode with BCFIPS. For now we need to disable
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponse.java
index e75d2c7ce2858..545837d7e5b87 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponse.java
@@ -25,7 +25,7 @@
 public class GetFeatureUpgradeStatusResponse {
 
     private static final ParseField FEATURE_UPGRADE_STATUSES = new ParseField("features");
-    private static final ParseField UPGRADE_STATUS = new ParseField("upgrade_status");
+    private static final ParseField UPGRADE_STATUS = new ParseField("migration_status");
 
     private final List<FeatureUpgradeStatus> featureUpgradeStatuses;
     private final String upgradeStatus;
@@ -76,7 +76,7 @@ public static class FeatureUpgradeStatus {
 
         private static final ParseField FEATURE_NAME = new ParseField("feature_name");
         private static final ParseField MINIMUM_INDEX_VERSION = new ParseField("minimum_index_version");
-        private static final ParseField UPGRADE_STATUS = new ParseField("upgrade_status");
+        private static final ParseField UPGRADE_STATUS = new ParseField("migration_status");
         private static final ParseField INDEX_VERSIONS = new ParseField("indices");
 
         @SuppressWarnings("unchecked")
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java
index 06ee1c97039f6..67d4818d26fc0 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java
@@ -23,8 +23,8 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
 
 public class MigrationIT extends ESRestHighLevelClientTestCase {
 
@@ -42,7 +42,7 @@ public void testGetDeprecationInfo() throws IOException {
     public void testGetFeatureUpgradeStatus() throws IOException {
         GetFeatureUpgradeStatusRequest request = new GetFeatureUpgradeStatusRequest();
         GetFeatureUpgradeStatusResponse response = highLevelClient().migration().getFeatureUpgradeStatus(request, RequestOptions.DEFAULT);
-        assertThat(response.getUpgradeStatus(), equalTo("NO_UPGRADE_NEEDED"));
+        assertThat(response.getUpgradeStatus(), equalTo("NO_MIGRATION_NEEDED"));
         assertThat(response.getFeatureUpgradeStatuses().size(), greaterThanOrEqualTo(1));
         Optional<GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus> optionalTasksStatus = response.getFeatureUpgradeStatuses().stream()
             .filter(status -> "tasks".equals(status.getFeatureName()))
@@ -52,7 +52,7 @@ public void testGetFeatureUpgradeStatus() throws IOException {
 
         GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus tasksStatus = optionalTasksStatus.get();
 
-        assertThat(tasksStatus.getUpgradeStatus(), equalTo("NO_UPGRADE_NEEDED"));
+        assertThat(tasksStatus.getUpgradeStatus(), equalTo("NO_MIGRATION_NEEDED"));
         assertThat(tasksStatus.getMinimumIndexVersion(), equalTo(Version.CURRENT.toString()));
         assertThat(tasksStatus.getFeatureName(), equalTo("tasks"));
     }
@@ -60,11 +60,8 @@ public void testGetFeatureUpgradeStatus() throws IOException {
     public void testPostFeatureUpgradeStatus() throws IOException {
         PostFeatureUpgradeRequest request = new PostFeatureUpgradeRequest();
         PostFeatureUpgradeResponse response = highLevelClient().migration().postFeatureUpgrade(request, RequestOptions.DEFAULT);
-        assertThat(response.isAccepted(), equalTo(true));
-        assertThat(response.getFeatures().size(), equalTo(1));
-        PostFeatureUpgradeResponse.Feature feature = response.getFeatures().get(0);
-        assertThat(feature.getFeatureName(), equalTo("security"));
-        assertThat(response.getReason(), nullValue());
-        assertThat(response.getElasticsearchException(), nullValue());
+        assertThat(response.isAccepted(), equalTo(false));
+        assertThat(response.getFeatures(), hasSize(0));
+        assertThat(response.getReason(), equalTo("No system indices require migration"));
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java
index 8099370e02fa9..5f5070d3f4a11 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.test.ESTestCase;
 
@@ -79,7 +80,7 @@ private static void toXContent(GetComponentTemplatesResponse response, XContentB
             builder.startObject();
             builder.field("name", e.getKey());
             builder.field("component_template");
-            e.getValue().toXContent(builder, null);
+            e.getValue().toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
         }
         builder.endArray();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java
index b93af425dd932..a241fb235b2b6 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -53,7 +54,7 @@ private static void toXContent(GetComposableIndexTemplatesResponse response, XCo
             builder.startObject();
             builder.field("name", e.getKey());
             builder.field("index_template");
-            e.getValue().toXContent(builder, null);
+            e.getValue().toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
         }
         builder.endArray();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponseTests.java
index f6483dd1651b5..7b3bb7cf6004b 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponseTests.java
@@ -40,10 +40,14 @@ protected org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStat
                     randomAlphaOfLengthBetween(3, 20),
                     randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()),
                     randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()),
-                    randomList(4,
-                        () -> new org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexVersion(
+                    randomList(
+                        4,
+                        () -> new org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexInfo(
                             randomAlphaOfLengthBetween(3, 20),
-                            randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion())))
+                            randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()),
+                            null
+                        )
+                    )
                 )),
             randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values())
         );
@@ -78,12 +82,12 @@ protected void assertInstances(
             assertThat(clientStatus.getIndexVersions(), hasSize(serverTestStatus.getIndexVersions().size()));
 
             for (int j = 0; i < clientStatus.getIndexVersions().size(); i++) {
-                org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexVersion serverIndexVersion
+                org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexInfo serverIndexInfo
                     = serverTestStatus.getIndexVersions().get(j);
                 GetFeatureUpgradeStatusResponse.IndexVersion clientIndexVersion = clientStatus.getIndexVersions().get(j);
 
-                assertThat(clientIndexVersion.getIndexName(), equalTo(serverIndexVersion.getIndexName()));
-                assertThat(clientIndexVersion.getVersion(), equalTo(serverIndexVersion.getVersion().toString()));
+                assertThat(clientIndexVersion.getIndexName(), equalTo(serverIndexInfo.getIndexName()));
+                assertThat(clientIndexVersion.getVersion(), equalTo(serverIndexInfo.getVersion().toString()));
             }
         }
     }
diff --git a/docs/reference/migration/apis/feature_upgrade.asciidoc b/docs/reference/migration/apis/feature_upgrade.asciidoc
index 88cd5d477f4d2..738227c5eae25 100644
--- a/docs/reference/migration/apis/feature_upgrade.asciidoc
+++ b/docs/reference/migration/apis/feature_upgrade.asciidoc
@@ -24,7 +24,7 @@ and to trigger an automated system upgrade that might potentially involve downti
 ==== {api-prereq-title}
 
 * If the {es} {security-features} are enabled, you must have the `manage`
-<<privileges-list-cluster,cluster privilege>> to use this API. (TODO: true?)
+<<privileges-list-cluster,cluster privilege>> to use this API.
 
 [[feature-upgrade-api-example]]
 ==== {api-examples-title}
@@ -46,79 +46,80 @@ Example response:
     {
       "feature_name" : "async_search",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "enrich",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "fleet",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "geoip",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "kibana",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "logstash_management",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "machine_learning",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "searchable_snapshots",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "security",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "tasks",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "transform",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "watcher",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     }
   ],
-  "upgrade_status" : "NO_UPGRADE_NEEDED"
+  "migration_status" : "NO_MIGRATION_NEEDED"
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"minimum_index_version" : "8.0.0"/"minimum_index_version" : $body.$_path/]
 
 This response tells us that Elasticsearch security needs its internal
 indices upgraded before we can upgrade the cluster to 8.0.
@@ -143,6 +144,7 @@ Example response:
   ]
 }
 --------------------------------------------------
+// TESTRESPONSE[skip: can't actually upgrade system indices in these tests]
 
 This tells us that the security index is being upgraded. To check the
 overall status of the upgrade, call the endpoint with GET.
diff --git a/docs/reference/migration/migrate_8_0.asciidoc b/docs/reference/migration/migrate_8_0.asciidoc
index e166a6eb83740..dad7b225b936a 100644
--- a/docs/reference/migration/migrate_8_0.asciidoc
+++ b/docs/reference/migration/migrate_8_0.asciidoc
@@ -25,7 +25,6 @@ coming[8.0.0]
 * <<breaking_80_ingest_changes>>
 * <<breaking_80_java_changes>>
 * <<breaking_80_mappings_changes>>
-* <<breaking_80_monitoring_changes>>
 * <<breaking_80_network_changes>>
 * <<breaking_80_node_changes>>
 * <<breaking_80_packaging_changes>>
@@ -128,7 +127,6 @@ include::migrate_8_0/indices.asciidoc[]
 include::migrate_8_0/ingest.asciidoc[]
 include::migrate_8_0/java.asciidoc[]
 include::migrate_8_0/mappings.asciidoc[]
-include::migrate_8_0/monitoring.asciidoc[]
 include::migrate_8_0/network.asciidoc[]
 include::migrate_8_0/node.asciidoc[]
 include::migrate_8_0/packaging.asciidoc[]
diff --git a/docs/reference/migration/migrate_8_0/monitoring.asciidoc b/docs/reference/migration/migrate_8_0/monitoring.asciidoc
deleted file mode 100644
index cdedfe29982c5..0000000000000
--- a/docs/reference/migration/migrate_8_0/monitoring.asciidoc
+++ /dev/null
@@ -1,47 +0,0 @@
-[discrete]
-[[breaking_80_monitoring_changes]]
-=== Monitoring changes
-
-//NOTE: The notable-breaking-changes tagged regions are re-used in the
-//Installation and Upgrade Guide
-
-//tag::notable-breaking-changes[]
-.The `use_ingest` setting on Monitoring exporter configurations has been removed.
-[%collapsible]
-====
-*Details* +
-The `xpack.monitoring.exporters.*.use_ingest` property was deprecated in 7.16.0 and
-has been removed. This parameter controlled the creation of pipelines for monitoring
-indices that previously had no function.
-
-*Impact* +
-Discontinue the use of the `xpack.monitoring.exporters.*.use_ingest` setting.
-====
-
-.The `index.pipeline.master_timeout` setting on Monitoring HTTP exporter configurations has been removed.
-[%collapsible]
-====
-*Details* +
-The `xpack.monitoring.exporters.*.index.pipeline.master_timeout` property was
-deprecated in 7.16.0. This parameter set the timeout when waiting for the remote
-Monitoring cluster to create pipelines. Those pipelines for monitoring indices previously
-had no function and are now removed in 8.0.0.
-
-*Impact* +
-Discontinue the use of the `xpack.monitoring.exporters.*.index.pipeline.master_timeout` setting.
-====
-
-.The `index.template.create_legacy_templates` setting on Monitoring HTTP exporter configurations has been removed.
-[%collapsible]
-====
-*Details* +
-The `xpack.monitoring.exporters.*.index.template.create_legacy_templates` property was
-deprecated in 7.16.0. This parameter instructed the exporter to install the previous version
-of monitoring templates on the monitoring cluster. These older templates were meant to assist
-in transitioning to the current monitoring data format. They are currently empty and are no
-longer of any use.
-
-*Impact* +
-Discontinue the use of the `xpack.monitoring.exporters.*.index.template.create_legacy_templates` setting.
-====
-//end::notable-breaking-changes[]
diff --git a/docs/reference/migration/migrate_8_0/security.asciidoc b/docs/reference/migration/migrate_8_0/security.asciidoc
index 3bda161e49161..d93fa300e0743 100644
--- a/docs/reference/migration/migrate_8_0/security.asciidoc
+++ b/docs/reference/migration/migrate_8_0/security.asciidoc
@@ -165,6 +165,37 @@ Specifying this setting in a transport profile in `elasticsearch.yml` will
 result in an error on startup.
 ====
 
+[discrete]
+[[saml-realm-nameid-changes]]
+.The `nameid_format` SAML realm setting no longer has a default value.
+[%collapsible]
+====
+*Details* +
+In SAML, Identity Providers (IdPs) can either be explicitly configured to
+release a `NameID` with a specific format, or configured to attempt to conform 
+with the requirements of a Service Provider (SP). The SP declares its
+requirements in the `NameIDPolicy` element of a SAML Authentication Request.
+In {es}, the `nameid_format` SAML realm setting controls the `NameIDPolicy`
+value.
+
+Previously, the default value for `nameid_format` was
+`urn:oasis:names:tc:SAML:2.0:nameid-format:transient`. This setting created
+authentication requests that required the IdP to release `NameID` with a
+`transient` format.
+
+The default value has been removed, which means that {es} will create SAML Authentication Requests by default that don't put this requirement on the
+IdP. If you want to retain the previous behavior, set `nameid_format` to
+`urn:oasis:names:tc:SAML:2.0:nameid-format:transient`.
+
+*Impact* +
+If you currently don't configure `nameid_format` explicitly, it's possible
+that your IdP will reject authentication requests from {es} because the requests 
+do not specify a `NameID` format (and your IdP is configured to expect one).
+This mismatch can result in a broken SAML configuration. If you're unsure whether 
+your IdP is explicitly configured to use a certain `NameID` format and you want to retain current behavior
+, try setting `nameid_format` to `urn:oasis:names:tc:SAML:2.0:nameid-format:transient` explicitly.
+====
+
 [discrete]
 [[ssl-validation-changes]]
 ===== SSL/TLS configuration validation
@@ -287,7 +318,7 @@ on startup.
 
 [discrete]
 [[ssl-misc-changes]]
-===== Other SSL/TLS changes 
+===== Other SSL/TLS changes
 
 .PKCS#11 keystores and trustores cannot be configured in `elasticsearch.yml`
 [%collapsible]
@@ -307,7 +338,7 @@ Use of a PKCS#11 keystore or truststore as the JRE's default store is not affect
 
 *Impact* +
 If you have a PKCS#11 keystore configured within your `elasticsearch.yml` file, you must remove that
-configuration and switch to a supported keystore type, or configure your PKCS#11 keystore as the 
+configuration and switch to a supported keystore type, or configure your PKCS#11 keystore as the
 JRE default store.
 ====
 
@@ -360,6 +391,7 @@ renamed to better reflect its intended use.
 Assign users with the `kibana_user` role to the `kibana_admin` role.
 Discontinue use of the `kibana_user` role.
 ====
+
 // end::notable-breaking-changes[]
 
 // These are non-notable changes
@@ -373,7 +405,7 @@ Discontinue use of the `kibana_user` role.
 [%collapsible]
 ====
 *Details* +
-If `xpack.security.fips_mode.enabled` is true (see <<fips-140-compliance>>), 
+If `xpack.security.fips_mode.enabled` is true (see <<fips-140-compliance>>),
 the value of `xpack.security.authc.password_hashing.algorithm` now defaults to
 `pbkdf2_stretch`.
 
diff --git a/docs/reference/transform/apis/index.asciidoc b/docs/reference/transform/apis/index.asciidoc
index b0948a7adb3dc..34a0b92c2d344 100644
--- a/docs/reference/transform/apis/index.asciidoc
+++ b/docs/reference/transform/apis/index.asciidoc
@@ -13,5 +13,6 @@ include::preview-transform.asciidoc[leveloffset=+2]
 include::start-transform.asciidoc[leveloffset=+2]
 //STOP
 include::stop-transform.asciidoc[leveloffset=+2]
-//UPDATE
-include::update-transform.asciidoc[leveloffset=+2]
\ No newline at end of file
+//UPDATE-UPGRADE
+include::update-transform.asciidoc[leveloffset=+2]
+include::upgrade-transforms.asciidoc[leveloffset=+2]
\ No newline at end of file
diff --git a/docs/reference/transform/apis/upgrade-transforms.asciidoc b/docs/reference/transform/apis/upgrade-transforms.asciidoc
index 7c95a31e63243..505f070a344b5 100644
--- a/docs/reference/transform/apis/upgrade-transforms.asciidoc
+++ b/docs/reference/transform/apis/upgrade-transforms.asciidoc
@@ -1,14 +1,14 @@
 [role="xpack"]
 [testenv="basic"]
 [[upgrade-transforms]]
-= Upgrade {transform} API
+= Upgrade {transforms} API
 
 [subs="attributes"]
 ++++
-<titleabbrev>Upgrade {transform}</titleabbrev>
+<titleabbrev>Upgrade {transforms}</titleabbrev>
 ++++
 
-Upgrades all {transform}s.
+Upgrades all {transforms}.
 
 [[upgrade-transforms-request]]
 == {api-request-title}
@@ -22,36 +22,68 @@ Requires the following privileges:
 
 * cluster: `manage_transform` (the `transform_admin` built-in role grants this
   privilege)
-* source indices: `read`, `view_index_metadata`
-* destination index: `read`, `index`.
 
 
 [[upgrade-transforms-desc]]
 == {api-description-title}
 
-This API upgrades all existing {transform}s.
+{transforms-cap} are compatible across minor versions and between supported 
+major versions. However, over time, the format of {transform} configuration 
+information may change. This API identifies {transforms} which have a legacy 
+configuration format and upgrades them to the latest version; including clean up 
+of the internal data structures that store {transform} state and checkpoints. 
+{transform-cap} upgrade does not effect the source and destination indices.
+
+If a {transform} upgrade step fails, the upgrade stops, and an error is returned 
+about the underlying issue. Resolve the issue then re-run the process again. A 
+summary is returned when the upgrade is finished.
+
+For a major version update – for example, from 7.16 to 8.0 –, it is recommended 
+to have a recent cluster backup prior to performing a {transform} upgrade which 
+can be run either before or after an {es} upgrade. However, it is recommended to 
+perform it before upgrading {es} to the next major version to ensure 
+{ctransforms} remain running.
+
+
+[IMPORTANT]
+====
+
+* When {es} {security-features} are enabled, your {transform} remembers the 
+roles of the user who created or updated it last. In contrast to 
+<<update-transform,update transform>>, a {transform} upgrade does not change the 
+stored roles, therefore the role used to read source data and write to the 
+destination index remains unchanged.
+
+====
+
 
 [[upgrade-transforms-query-parms]]
 == {api-query-parms-title}
 
 `dry_run`::
-  (Optional, Boolean) When `true`, only checks for updates but does not execute them.
+  (Optional, Boolean) When `true`, only checks for updates but does not execute 
+  them. Defaults to `false`.
+
 
 [[upgrade-transforms-example]]
 == {api-examples-title}
 
+To upgrade the legacy {transforms} to the latest configuration format, perform 
+the following API call:
+
 [source,console]
 --------------------------------------------------
 POST _transform/_upgrade
 --------------------------------------------------
 // TEST[setup:simple_kibana_continuous_pivot]
 
-When all {transform}s are upgraded, you receive a summary:
+When all {transforms} are upgraded, you receive a summary:
 
 [source,console-result]
 ----
 {
+  "updated": 2,
   "no_action": 1
 }
 ----
-// TESTRESPONSE[s/"no_action" : 1/"no_action" : $body.no_action/]
+// TESTRESPONSE[skip:TBD]
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..d92d8f3fc6b2f
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+93676b57287ef17b5eda8f0ab5aa91ac27781b30
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index fe9352ca233c8..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3bc34dea0b46e0f6429b054f848e2611d6e1d3e7
\ No newline at end of file
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt
index 61986fab9423c..af330de477216 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt
@@ -13,9 +13,6 @@ class org.elasticsearch.script.field.Field @dynamic_type {
   String getName()
   boolean isEmpty()
   int size()
-  List getValues()
-  def getValue(def)
-  def getValue(int, def)
 }
 
 class org.elasticsearch.script.DocBasedScript {
diff --git a/modules/legacy-geo/licenses/lucene-spatial-extras-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/modules/legacy-geo/licenses/lucene-spatial-extras-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..a3ea1b0fef522
--- /dev/null
+++ b/modules/legacy-geo/licenses/lucene-spatial-extras-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+fabf62d18d45d4eb4e6199005f92624233b3f2e4
\ No newline at end of file
diff --git a/modules/legacy-geo/licenses/lucene-spatial-extras-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/modules/legacy-geo/licenses/lucene-spatial-extras-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 1c96b8e73ff65..0000000000000
--- a/modules/legacy-geo/licenses/lucene-spatial-extras-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d9f8584667012bf09c446512ee25191b5d00ceed
\ No newline at end of file
diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java
new file mode 100644
index 0000000000000..09765b472c770
--- /dev/null
+++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java
@@ -0,0 +1,388 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.migration;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction;
+import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusRequest;
+import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse;
+import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction;
+import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeRequest;
+import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SystemIndexPlugin;
+import org.elasticsearch.reindex.ReindexPlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.upgrades.FeatureMigrationResults;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class FeatureMigrationIT extends ESIntegTestCase {
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)).build();
+    }
+
+    @Override
+    protected boolean forbidPrivateIndexSettings() {
+        // We need to be able to set the index creation version manually.
+        return false;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(TestPlugin.class);
+        plugins.add(ReindexPlugin.class);
+        return plugins;
+    }
+
+    public void testMigrateInternalManagedSystemIndex() throws Exception {
+        createSystemIndexForDescriptor(INTERNAL_MANAGED);
+        createSystemIndexForDescriptor(INTERNAL_UNMANAGED);
+        createSystemIndexForDescriptor(EXTERNAL_MANAGED);
+        createSystemIndexForDescriptor(EXTERNAL_UNMANAGED);
+
+        ensureGreen();
+
+        SetOnce<Boolean> preUpgradeHookCalled = new SetOnce<>();
+        SetOnce<Boolean> postUpgradeHookCalled = new SetOnce<>();
+        TestPlugin.preMigrationHook.set(clusterState -> {
+            // Check that the ordering of these calls is correct.
+            assertThat(postUpgradeHookCalled.get(), nullValue());
+            Map<String, Object> metadata = new HashMap<>();
+            metadata.put("stringKey", "stringValue");
+            metadata.put("intKey", 42);
+            {
+                Map<String, Object> innerMetadata = new HashMap<>();
+                innerMetadata.put("innerKey", "innerValue");
+
+                metadata.put("mapKey", innerMetadata);
+            }
+            metadata.put("listKey", Arrays.asList(1, 2, 3, 4));
+            preUpgradeHookCalled.set(true);
+            return metadata;
+        });
+
+        TestPlugin.postMigrationHook.set((clusterState, metadata) -> {
+            assertThat(preUpgradeHookCalled.get(), is(true));
+
+            assertThat(
+                metadata,
+                hasEntry("stringKey", "stringValue")
+            );
+            assertThat(metadata, hasEntry("intKey", 42));
+            assertThat(metadata, hasEntry("listKey", Arrays.asList(1,2,3,4)));
+            assertThat(metadata, hasKey("mapKey"));
+            @SuppressWarnings("unchecked")
+            Map<String, Object> innerMap = (Map<String, Object>) metadata.get("mapKey");
+            assertThat(innerMap, hasEntry("innerKey", "innerValue"));
+
+            // We shouldn't have any results in the cluster state as no features have fully finished yet.
+            FeatureMigrationResults currentResults = clusterState.metadata().custom(FeatureMigrationResults.TYPE);
+            assertThat(currentResults, nullValue());
+            postUpgradeHookCalled.set(true);
+        });
+
+        PostFeatureUpgradeRequest migrationRequest = new PostFeatureUpgradeRequest();
+        PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get();
+        assertThat(migrationResponse.getReason(), nullValue());
+        assertThat(migrationResponse.getElasticsearchException(), nullValue());
+        final Set<String> migratingFeatures = migrationResponse.getFeatures()
+            .stream()
+            .map(PostFeatureUpgradeResponse.Feature::getFeatureName)
+            .collect(Collectors.toSet());
+        assertThat(migratingFeatures, hasItem(FEATURE_NAME));
+
+        GetFeatureUpgradeStatusRequest getStatusRequest = new GetFeatureUpgradeStatusRequest();
+        assertBusy(() -> {
+            GetFeatureUpgradeStatusResponse statusResponse = client().execute(GetFeatureUpgradeStatusAction.INSTANCE, getStatusRequest)
+                .get();
+            logger.info(Strings.toString(statusResponse));
+            assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED));
+        });
+
+        assertTrue("the pre-migration hook wasn't actually called", preUpgradeHookCalled.get());
+        assertTrue("the post-migration hook wasn't actually called", postUpgradeHookCalled.get());
+
+        Metadata finalMetadata = client().admin().cluster().prepareState().get().getState().metadata();
+        // Check that the results metadata is what we expect.
+        FeatureMigrationResults currentResults = finalMetadata.custom(FeatureMigrationResults.TYPE);
+        assertThat(currentResults, notNullValue());
+        assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(FEATURE_NAME)));
+        assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).succeeded(), is(true));
+        assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedIndexName(), nullValue());
+        assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getException(), nullValue());
+
+        assertIndexHasCorrectProperties(
+            finalMetadata,
+            ".int-man-old-reindexed-for-8",
+            INTERNAL_MANAGED_FLAG_VALUE,
+            true,
+            true,
+            Arrays.asList(".int-man-old", ".internal-managed-alias")
+        );
+        assertIndexHasCorrectProperties(
+            finalMetadata,
+            ".int-unman-old-reindexed-for-8",
+            INTERNAL_UNMANAGED_FLAG_VALUE,
+            false,
+            true,
+            Collections.singletonList(".int-unman-old")
+        );
+        assertIndexHasCorrectProperties(
+            finalMetadata,
+            ".ext-man-old-reindexed-for-8",
+            EXTERNAL_MANAGED_FLAG_VALUE,
+            true,
+            false,
+            Arrays.asList(".ext-man-old", ".external-managed-alias")
+        );
+        assertIndexHasCorrectProperties(
+            finalMetadata,
+            ".ext-unman-old-reindexed-for-8",
+            EXTERNAL_UNMANAGED_FLAG_VALUE,
+            false,
+            false,
+            Collections.singletonList(".ext-unman-old")
+        );
+    }
+
+    public void assertIndexHasCorrectProperties(
+        Metadata metadata,
+        String indexName,
+        int settingsFlagValue,
+        boolean isManaged,
+        boolean isInternal,
+        Collection<String> aliasNames) {
+        IndexMetadata imd = metadata.index(indexName);
+        assertThat(imd.getSettings().get(FlAG_SETTING_KEY), equalTo(Integer.toString(settingsFlagValue)));
+        final Map<String, Object> mapping = imd.mapping().getSourceAsMap();
+        @SuppressWarnings("unchecked")
+        final Map<String, Object> meta = (Map<String, Object>) mapping.get("_meta");
+        assertThat(meta.get(DESCRIPTOR_MANAGED_META_KEY), is(isManaged));
+        assertThat(meta.get(DESCRIPTOR_INTERNAL_META_KEY), is(isInternal));
+
+        assertThat(imd.isSystem(), is(true));
+
+        Set<String> actualAliasNames = imd.getAliases().keySet();
+        assertThat(actualAliasNames, containsInAnyOrder(aliasNames.toArray()));
+
+        IndicesStatsResponse indexStats = client().admin().indices().prepareStats(imd.getIndex().getName()).setDocs(true).get();
+        assertThat(indexStats.getIndex(imd.getIndex().getName()).getTotal().getDocs().getCount(), is((long) INDEX_DOC_COUNT));
+    }
+
+    public void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) throws InterruptedException {
+        assertTrue(
+            "the strategy used below to create index names for descriptors without a primary index name only works for simple patterns",
+            descriptor.getIndexPattern().endsWith("*")
+        );
+        String indexName = Optional.ofNullable(descriptor.getPrimaryIndex()).orElse(descriptor.getIndexPattern().replace("*", "old"));
+        CreateIndexRequestBuilder createRequest = prepareCreate(indexName);
+        createRequest.setWaitForActiveShards(ActiveShardCount.ALL);
+        if (descriptor.getSettings() != null) {
+            createRequest.setSettings(Settings.builder().put("index.version.created", Version.CURRENT).build());
+        } else {
+            createRequest.setSettings(
+                createSimpleSettings(
+                    Version.V_7_0_0,
+                    descriptor.isInternal() ? INTERNAL_UNMANAGED_FLAG_VALUE : EXTERNAL_UNMANAGED_FLAG_VALUE
+                )
+            );
+        }
+        if (descriptor.getMappings() == null) {
+            createRequest.setMapping(createSimpleMapping(false, descriptor.isInternal()));
+        }
+        CreateIndexResponse response = createRequest.get();
+        assertTrue(response.isShardsAcknowledged());
+
+        List<IndexRequestBuilder> docs = new ArrayList<>(INDEX_DOC_COUNT);
+        for (int i = 0; i < INDEX_DOC_COUNT; i++) {
+            docs.add(client().prepareIndex(indexName).setId(Integer.toString(i)).setSource("some_field", "words words"));
+        }
+        indexRandom(true, docs);
+        IndicesStatsResponse indexStats = client().admin().indices().prepareStats(indexName).setDocs(true).get();
+        assertThat(indexStats.getIndex(indexName).getTotal().getDocs().getCount(), is((long) INDEX_DOC_COUNT));
+    }
+
+    static final String VERSION_META_KEY = "version";
+    static final Version META_VERSION = Version.CURRENT;
+    static final String DESCRIPTOR_MANAGED_META_KEY = "desciptor_managed";
+    static final String DESCRIPTOR_INTERNAL_META_KEY = "descriptor_internal";
+    static final String FEATURE_NAME = "A-test-feature"; // Sorts alphabetically before the feature from MultiFeatureMigrationIT
+    static final String ORIGIN = FeatureMigrationIT.class.getSimpleName();
+    static final String FlAG_SETTING_KEY = IndexMetadata.INDEX_PRIORITY_SETTING.getKey();
+    static final int INDEX_DOC_COUNT = 100; // arbitrarily chosen
+
+    static final int INTERNAL_MANAGED_FLAG_VALUE = 1;
+    static final int INTERNAL_UNMANAGED_FLAG_VALUE = 2;
+    static final int EXTERNAL_MANAGED_FLAG_VALUE = 3;
+    static final int EXTERNAL_UNMANAGED_FLAG_VALUE = 4;
+    static final SystemIndexDescriptor INTERNAL_MANAGED = SystemIndexDescriptor.builder()
+        .setIndexPattern(".int-man-*")
+        .setAliasName(".internal-managed-alias")
+        .setPrimaryIndex(".int-man-old")
+        .setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED)
+        .setSettings(createSimpleSettings(Version.V_7_0_0, INTERNAL_MANAGED_FLAG_VALUE))
+        .setMappings(createSimpleMapping(true, true))
+        .setOrigin(ORIGIN)
+        .setVersionMetaKey(VERSION_META_KEY)
+        .setAllowedElasticProductOrigins(Collections.emptyList())
+        .setMinimumNodeVersion(Version.V_7_0_0)
+        .setPriorSystemIndexDescriptors(Collections.emptyList())
+        .build();
+    static final SystemIndexDescriptor INTERNAL_UNMANAGED = SystemIndexDescriptor.builder()
+        .setIndexPattern(".int-unman-*")
+        .setType(SystemIndexDescriptor.Type.INTERNAL_UNMANAGED)
+        .setOrigin(ORIGIN)
+        .setVersionMetaKey(VERSION_META_KEY)
+        .setAllowedElasticProductOrigins(Collections.emptyList())
+        .setMinimumNodeVersion(Version.V_7_0_0)
+        .setPriorSystemIndexDescriptors(Collections.emptyList())
+        .build();
+    static final SystemIndexDescriptor EXTERNAL_MANAGED = SystemIndexDescriptor.builder()
+        .setIndexPattern(".ext-man-*")
+        .setAliasName(".external-managed-alias")
+        .setPrimaryIndex(".ext-man-old")
+        .setType(SystemIndexDescriptor.Type.EXTERNAL_MANAGED)
+        .setSettings(createSimpleSettings(Version.V_7_0_0, EXTERNAL_MANAGED_FLAG_VALUE))
+        .setMappings(createSimpleMapping(true, false))
+        .setOrigin(ORIGIN)
+        .setVersionMetaKey(VERSION_META_KEY)
+        .setAllowedElasticProductOrigins(Collections.singletonList(ORIGIN))
+        .setMinimumNodeVersion(Version.V_7_0_0)
+        .setPriorSystemIndexDescriptors(Collections.emptyList())
+        .build();
+    static final SystemIndexDescriptor EXTERNAL_UNMANAGED = SystemIndexDescriptor.builder()
+        .setIndexPattern(".ext-unman-*")
+        .setType(SystemIndexDescriptor.Type.EXTERNAL_UNMANAGED)
+        .setOrigin(ORIGIN)
+        .setVersionMetaKey(VERSION_META_KEY)
+        .setAllowedElasticProductOrigins(Collections.singletonList(ORIGIN))
+        .setMinimumNodeVersion(Version.V_7_0_0)
+        .setPriorSystemIndexDescriptors(Collections.emptyList())
+        .build();
+
+    static Settings createSimpleSettings(Version creationVersion, int flagSettingValue) {
+        return Settings.builder()
+            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+            .put(FlAG_SETTING_KEY, flagSettingValue)
+            .put("index.version.created", creationVersion)
+            .build();
+    }
+
+    static String createSimpleMapping(boolean descriptorManaged, boolean descriptorInternal) {
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.startObject();
+            {
+                builder.startObject("_meta");
+                builder.field(VERSION_META_KEY, META_VERSION);
+                builder.field(DESCRIPTOR_MANAGED_META_KEY, descriptorManaged);
+                builder.field(DESCRIPTOR_INTERNAL_META_KEY, descriptorInternal);
+                builder.endObject();
+
+                builder.field("dynamic", "strict");
+                builder.startObject("properties");
+                {
+                    builder.startObject("some_field");
+                    builder.field("type", "keyword");
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            return Strings.toString(builder);
+        } catch (IOException e) {
+            // Just rethrow, it should be impossible for this to throw here
+            throw new AssertionError(e);
+        }
+    }
+
+    public static class TestPlugin extends Plugin implements SystemIndexPlugin {
+        public static final AtomicReference<Function<ClusterState, Map<String, Object>>> preMigrationHook = new AtomicReference<>();
+        public static final AtomicReference<BiConsumer<ClusterState, Map<String, Object>>> postMigrationHook = new AtomicReference<>();
+
+        public TestPlugin() {
+
+        }
+
+        @Override
+        public String getFeatureName() {
+            return FEATURE_NAME;
+        }
+
+        @Override
+        public String getFeatureDescription() {
+            return "a plugin for testing system index migration";
+        }
+
+        @Override
+        public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+            return Arrays.asList(INTERNAL_MANAGED, INTERNAL_UNMANAGED, EXTERNAL_MANAGED, EXTERNAL_UNMANAGED);
+        }
+
+        @Override
+        public void prepareForIndicesMigration(ClusterService clusterService, Client client, ActionListener<Map<String, Object>> listener) {
+            listener.onResponse(preMigrationHook.get().apply(clusterService.state()));
+        }
+
+        @Override
+        public void indicesMigrationComplete(
+            Map<String, Object> preUpgradeMetadata,
+            ClusterService clusterService,
+            Client client,
+            ActionListener<Boolean> listener
+        ) {
+            postMigrationHook.get().accept(clusterService.state(), preUpgradeMetadata);
+            listener.onResponse(true);
+        }
+    }
+}
diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java
new file mode 100644
index 0000000000000..832990a973e53
--- /dev/null
+++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java
@@ -0,0 +1,302 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.migration;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction;
+import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusRequest;
+import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse;
+import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction;
+import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeRequest;
+import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SystemIndexPlugin;
+import org.elasticsearch.reindex.ReindexPlugin;
+import org.elasticsearch.upgrades.FeatureMigrationResults;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class MultiFeatureMigrationIT extends FeatureMigrationIT {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)).build();
+    }
+
+    @Override
+    protected boolean forbidPrivateIndexSettings() {
+        // We need to be able to set the index creation version manually.
+        return false;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(FeatureMigrationIT.TestPlugin.class);
+        plugins.add(SecondPlugin.class);
+        plugins.add(ReindexPlugin.class);
+        return plugins;
+    }
+
+    // Sorts alphabetically after the feature from MultiFeatureMigrationIT
+    private static final String SECOND_FEATURE_NAME = "B-test-feature";
+    private static final String ORIGIN = MultiFeatureMigrationIT.class.getSimpleName();
+    private static final String VERSION_META_KEY = "version";
+    static final int SECOND_FEATURE_IDX_FLAG_VALUE = 0;
+
+    public void testMultipleFeatureMigration() throws Exception {
+        // All the indices from FeatureMigrationIT
+        createSystemIndexForDescriptor(INTERNAL_MANAGED);
+        createSystemIndexForDescriptor(INTERNAL_UNMANAGED);
+        createSystemIndexForDescriptor(EXTERNAL_MANAGED);
+        createSystemIndexForDescriptor(EXTERNAL_UNMANAGED);
+        // And our new one
+        createSystemIndexForDescriptor(SECOND_FEATURE_IDX_DESCIPTOR);
+
+        ensureGreen();
+
+        SetOnce<Boolean> preMigrationHookCalled = new SetOnce<>();
+        SetOnce<Boolean> postMigrationHookCalled = new SetOnce<>();
+        SetOnce<Boolean> secondPluginPreMigrationHookCalled = new SetOnce<>();
+        SetOnce<Boolean> secondPluginPostMigrationHookCalled = new SetOnce<>();
+
+        TestPlugin.preMigrationHook.set(clusterState -> {
+            // None of the other hooks should have been called yet.
+            assertThat(postMigrationHookCalled.get(), nullValue());
+            assertThat(secondPluginPreMigrationHookCalled.get(), nullValue());
+            assertThat(secondPluginPostMigrationHookCalled.get(), nullValue());
+            Map<String, Object> metadata = new HashMap<>();
+            metadata.put("stringKey", "first plugin value");
+
+            // We shouldn't have any results in the cluster state given no features have finished yet.
+            FeatureMigrationResults currentResults = clusterState.metadata().custom(FeatureMigrationResults.TYPE);
+            assertThat(currentResults, nullValue());
+
+            preMigrationHookCalled.set(true);
+            return metadata;
+        });
+
+        TestPlugin.postMigrationHook.set((clusterState, metadata) -> {
+            // Check that the hooks have been called or not as expected.
+            assertThat(preMigrationHookCalled.get(), is(true));
+            assertThat(secondPluginPreMigrationHookCalled.get(), nullValue());
+            assertThat(secondPluginPostMigrationHookCalled.get(), nullValue());
+
+            assertThat(
+                metadata,
+                hasEntry("stringKey", "first plugin value")
+            );
+
+            // We shouldn't have any results in the cluster state given no features have finished yet.
+            FeatureMigrationResults currentResults = clusterState.metadata().custom(FeatureMigrationResults.TYPE);
+            assertThat(currentResults, nullValue());
+
+            postMigrationHookCalled.set(true);
+        });
+
+        SecondPlugin.preMigrationHook.set(clusterState -> {
+            // Check that the hooks have been called or not as expected.
+            assertThat(preMigrationHookCalled.get(), is(true));
+            assertThat(postMigrationHookCalled.get(), is(true));
+            assertThat(secondPluginPostMigrationHookCalled.get(), nullValue());
+
+            Map<String, Object> metadata = new HashMap<>();
+            metadata.put("stringKey", "second plugin value");
+
+            // But now, we should have results, as we're in a new feature!
+            FeatureMigrationResults currentResults = clusterState.metadata().custom(FeatureMigrationResults.TYPE);
+            assertThat(currentResults, notNullValue());
+            assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(FEATURE_NAME)));
+            assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).succeeded(), is(true));
+            assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedIndexName(), nullValue());
+            assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getException(), nullValue());
+
+            secondPluginPreMigrationHookCalled.set(true);
+            return metadata;
+        });
+
+        SecondPlugin.postMigrationHook.set((clusterState, metadata) -> {
+            // Check that the hooks have been called or not as expected.
+            assertThat(preMigrationHookCalled.get(), is(true));
+            assertThat(postMigrationHookCalled.get(), is(true));
+            assertThat(secondPluginPreMigrationHookCalled.get(), is(true));
+
+            assertThat(
+                metadata,
+                hasEntry("stringKey", "second plugin value")
+            );
+
+            // And here, the results should be the same, as we haven't updated the state with this feature's status yet.
+            FeatureMigrationResults currentResults = clusterState.metadata().custom(FeatureMigrationResults.TYPE);
+            assertThat(currentResults, notNullValue());
+            assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(FEATURE_NAME)));
+            assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).succeeded(), is(true));
+            assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedIndexName(), nullValue());
+            assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getException(), nullValue());
+
+            secondPluginPostMigrationHookCalled.set(true);
+        });
+
+        PostFeatureUpgradeRequest migrationRequest = new PostFeatureUpgradeRequest();
+        PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get();
+        assertThat(migrationResponse.getReason(), nullValue());
+        assertThat(migrationResponse.getElasticsearchException(), nullValue());
+        final Set<String> migratingFeatures = migrationResponse.getFeatures()
+            .stream()
+            .map(PostFeatureUpgradeResponse.Feature::getFeatureName)
+            .collect(Collectors.toSet());
+        assertThat(migratingFeatures, hasItems(FEATURE_NAME, SECOND_FEATURE_NAME));
+
+        GetFeatureUpgradeStatusRequest getStatusRequest = new GetFeatureUpgradeStatusRequest();
+        assertBusy(() -> {
+            GetFeatureUpgradeStatusResponse statusResponse = client().execute(GetFeatureUpgradeStatusAction.INSTANCE, getStatusRequest)
+                .get();
+            logger.info(Strings.toString(statusResponse));
+            assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED));
+        });
+
+        assertTrue("the first plugin's pre-migration hook wasn't actually called", preMigrationHookCalled.get());
+        assertTrue("the first plugin's post-migration hook wasn't actually called", postMigrationHookCalled.get());
+
+        assertTrue("the second plugin's pre-migration hook wasn't actually called", secondPluginPreMigrationHookCalled.get());
+        assertTrue("the second plugin's post-migration hook wasn't actually called", secondPluginPostMigrationHookCalled.get());
+
+        Metadata finalMetadata = client().admin().cluster().prepareState().get().getState().metadata();
+        // Check that the results metadata is what we expect
+        FeatureMigrationResults currentResults = finalMetadata.custom(FeatureMigrationResults.TYPE);
+        assertThat(currentResults, notNullValue());
+        assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(2), hasKey(FEATURE_NAME), hasKey(SECOND_FEATURE_NAME)));
+        assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).succeeded(), is(true));
+        assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedIndexName(), nullValue());
+        assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getException(), nullValue());
+        assertThat(currentResults.getFeatureStatuses().get(SECOND_FEATURE_NAME).succeeded(), is(true));
+        assertThat(currentResults.getFeatureStatuses().get(SECOND_FEATURE_NAME).getFailedIndexName(), nullValue());
+        assertThat(currentResults.getFeatureStatuses().get(SECOND_FEATURE_NAME).getException(), nullValue());
+
+        // Finally, verify that all the indices exist and have the properties we expect.
+        assertIndexHasCorrectProperties(
+            finalMetadata,
+            ".int-man-old-reindexed-for-8",
+            INTERNAL_MANAGED_FLAG_VALUE,
+            true,
+            true,
+            Arrays.asList(".int-man-old", ".internal-managed-alias")
+        );
+        assertIndexHasCorrectProperties(
+            finalMetadata,
+            ".int-unman-old-reindexed-for-8",
+            INTERNAL_UNMANAGED_FLAG_VALUE,
+            false,
+            true,
+            Collections.singletonList(".int-unman-old")
+        );
+        assertIndexHasCorrectProperties(
+            finalMetadata,
+            ".ext-man-old-reindexed-for-8",
+            EXTERNAL_MANAGED_FLAG_VALUE,
+            true,
+            false,
+            Arrays.asList(".ext-man-old", ".external-managed-alias")
+        );
+        assertIndexHasCorrectProperties(
+            finalMetadata,
+            ".ext-unman-old-reindexed-for-8",
+            EXTERNAL_UNMANAGED_FLAG_VALUE,
+            false,
+            false,
+            Collections.singletonList(".ext-unman-old")
+        );
+
+        assertIndexHasCorrectProperties(
+            finalMetadata,
+            ".second-int-man-old-reindexed-for-8",
+            SECOND_FEATURE_IDX_FLAG_VALUE,
+            true,
+            true,
+            Arrays.asList(".second-int-man-old", ".second-internal-managed-alias")
+        );
+    }
+
+    private static final SystemIndexDescriptor SECOND_FEATURE_IDX_DESCIPTOR = SystemIndexDescriptor.builder()
+        .setIndexPattern(".second-int-man-*")
+        .setAliasName(".second-internal-managed-alias")
+        .setPrimaryIndex(".second-int-man-old")
+        .setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED)
+        .setSettings(createSimpleSettings(Version.V_7_0_0, 0))
+        .setMappings(createSimpleMapping(true, true))
+        .setOrigin(ORIGIN)
+        .setVersionMetaKey(VERSION_META_KEY)
+        .setAllowedElasticProductOrigins(Collections.emptyList())
+        .setMinimumNodeVersion(Version.V_7_0_0)
+        .setPriorSystemIndexDescriptors(Collections.emptyList())
+        .build();
+
+    public static class SecondPlugin extends Plugin implements SystemIndexPlugin {
+
+        private static final AtomicReference<Function<ClusterState, Map<String, Object>>> preMigrationHook = new AtomicReference<>();
+        private static final AtomicReference<BiConsumer<ClusterState, Map<String, Object>>> postMigrationHook = new AtomicReference<>();
+
+        public SecondPlugin() {
+
+        }
+
+        @Override public String getFeatureName() {
+            return SECOND_FEATURE_NAME;
+        }
+
+        @Override public String getFeatureDescription() {
+            return "a plugin for test system index migration with multiple features";
+        }
+
+        @Override public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+            return Collections.singletonList(SECOND_FEATURE_IDX_DESCIPTOR);
+        }
+
+        @Override public void prepareForIndicesMigration(
+            ClusterService clusterService, Client client, ActionListener<Map<String, Object>> listener) {
+            listener.onResponse(preMigrationHook.get().apply(clusterService.state()));
+        }
+
+        @Override public void indicesMigrationComplete(
+            Map<String, Object> preUpgradeMetadata, ClusterService clusterService, Client client, ActionListener<Boolean> listener) {
+            postMigrationHook.get().accept(clusterService.state(), preUpgradeMetadata);
+            listener.onResponse(true);
+        }
+    }
+}
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..19406d09ecf18
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+c5b251c26358225271e6901ae8e62f2b7b9a6e98
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 8c6f53bfae64d..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-91d7bb3e0f54577efbd08f8527ab705f76b405be
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..f40fc403c5530
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+07e69528347205991aa8b9724170c6bb4a61b0b0
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index dc92541178560..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2e37f30126a2ddc0231b929b82b2bc2e120af1c2
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..3540eb801f1ae
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+f170e5a8a8105c7eb747ab77ee3a91247f1704b1
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 0a00260a3592c..0000000000000
--- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-efae6bf6515c2d767491b85a50946279f66e9836
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..308d0978017df
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+e1c03aa49d6dc9c5f194446ba1ac39504628456e
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index cf21023f7195d..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e50a65b19d22bc98ca57a9426c45138c71787154
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..0e0de95bafd14
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+62b1dcdb25704edf95e634a810f8aa36aaac99ab
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 4bfee0a4dabe1..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6de256a78d4c8838d70f4b3720ba9fb6242a20bf
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..0fb07bd6f7352
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+10cf350a47eb30b615a7fd9bf740ad5e45017a2d
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 2c7e1a52098af..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-48030e67171008e5d76fb211091a2bd4d256f097
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..04a4d5df07ded
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+839cd3dcb679d53239860f88a70c970a4b30c93a
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 2d8b0dc4dd2a5..0000000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-21a775f1b3a2912c02ac64ceb23746b259694938
\ No newline at end of file
diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java
index aaaafd5c15c8c..49fa8e8dd46c9 100644
--- a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java
+++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java
@@ -103,6 +103,7 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 
 /**
@@ -438,7 +439,6 @@ public void testSortByFieldOneClusterHasNoResults() throws Exception {
         assumeMultiClusterSetup();
         SearchRequest searchRequest = initSearchRequest();
         // set to a value greater than the number of shards to avoid differences due to the skipping of shards
-        searchRequest.setPreFilterShardSize(128);
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
         boolean onlyRemote = randomBoolean();
         sourceBuilder.query(new TermQueryBuilder("_index", onlyRemote ? REMOTE_INDEX_NAME : INDEX_NAME));
@@ -461,7 +461,6 @@ public void testSortByFieldOneClusterHasNoResults() throws Exception {
         });
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/79365")
     public void testFieldCollapsingOneClusterHasNoResults() throws Exception {
         assumeMultiClusterSetup();
         SearchRequest searchRequest = initSearchRequest();
@@ -771,6 +770,7 @@ private static void duelSearch(SearchRequest searchRequest, Consumer<SearchRespo
                 message.compareMaps(minimizeRoundtripsResponseMap, fanOutResponseMap);
                 throw new AssertionError("Didn't match expected value:\n" + message);
             }
+            assertThat(minimizeRoundtripsSearchResponse.getSkippedShards(), lessThanOrEqualTo(fanOutSearchResponse.getSkippedShards()));
         }
     }
 
@@ -835,6 +835,10 @@ private static Map<String, Object> responseToMap(SearchResponse response) throws
                 shard.remove("fetch");
             }
         }
+        Map<String, Object> shards = (Map<String, Object>)responseMap.get("_shards");
+        if (shards != null) {
+            shards.remove("skipped");
+        }
         return responseMap;
     }
 
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java
index 8bba5325cec8d..0591e0521ff78 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java
@@ -91,10 +91,10 @@ public void testGetFeatureUpgradeStatus() throws Exception {
 
                 assertThat(feature.size(), equalTo(4));
                 assertThat(feature.get("minimum_index_version"), equalTo(UPGRADE_FROM_VERSION.toString()));
-                if (UPGRADE_FROM_VERSION.before(Version.CURRENT.minimumIndexCompatibilityVersion())) {
-                    assertThat(feature.get("upgrade_status"), equalTo("UPGRADE_NEEDED"));
+                if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) {
+                    assertThat(feature.get("migration_status"), equalTo("MIGRATION_NEEDED"));
                 } else {
-                    assertThat(feature.get("upgrade_status"), equalTo("NO_UPGRADE_NEEDED"));
+                    assertThat(feature.get("migration_status"), equalTo("NO_MIGRATION_NEEDED"));
                 }
             });
         }
diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java
index af9839a772a10..32e5abfb9a3e0 100644
--- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java
+++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java
@@ -52,8 +52,8 @@ public void testGetFeatureUpgradedStatuses() throws Exception {
         Response response = client().performRequest(new Request("GET", "/_migration/system_features"));
         assertThat(response.getStatusLine().getStatusCode(), is(200));
         XContentTestUtils.JsonMapView view = XContentTestUtils.createJsonMapView(response.getEntity().getContent());
-        String upgradeStatus = view.get("upgrade_status");
-        assertThat(upgradeStatus, equalTo("NO_UPGRADE_NEEDED"));
+        String upgradeStatus = view.get("migration_status");
+        assertThat(upgradeStatus, equalTo("NO_MIGRATION_NEEDED"));
         List<Map<String, Object>> features = view.get("features");
         Map<String, Object> testFeature = features.stream()
             .filter(feature -> "system indices qa".equals(feature.get("feature_name")))
@@ -62,7 +62,7 @@ public void testGetFeatureUpgradedStatuses() throws Exception {
 
         assertThat(testFeature.size(), equalTo(4));
         assertThat(testFeature.get("minimum_index_version"), equalTo(Version.CURRENT.toString()));
-        assertThat(testFeature.get("upgrade_status"), equalTo("NO_UPGRADE_NEEDED"));
+        assertThat(testFeature.get("migration_status"), equalTo("NO_MIGRATION_NEEDED"));
         assertThat(testFeature.get("indices"), instanceOf(List.class));
 
         assertThat((List<Object>) testFeature.get("indices"), hasSize(1));
diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle
index 0cef59a464047..0f26777f04ec7 100644
--- a/rest-api-spec/build.gradle
+++ b/rest-api-spec/build.gradle
@@ -81,6 +81,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task ->
   task.skipTest("search.aggregation/20_terms/string profiler via global ordinals native implementation", "The profiler results aren't backwards compatible.")
   task.skipTest("search.aggregation/20_terms/string profiler via map", "The profiler results aren't backwards compatible.")
   task.skipTest("search.aggregation/20_terms/numeric profiler", "The profiler results aren't backwards compatible.")
+  task.skipTest("migration/10_get_feature_upgrade_status/Get feature upgrade status", "Awaits backport")
 
   task.replaceValueInMatch("_type", "_doc")
   task.addAllowedWarningRegex("\\[types removal\\].*")
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml
index 1fd5bc9c85a24..fefa4bb48230b 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml
@@ -7,5 +7,5 @@
   - do:
       migration.get_feature_upgrade_status: {}
 
-  - is_true: upgrade_status
+  - is_true: migration_status
   - is_true: features
diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..1d5734eb988f7
--- /dev/null
+++ b/server/licenses/lucene-analysis-common-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+0f73e4a59da39da41511bb960212398d82e64a97
\ No newline at end of file
diff --git a/server/licenses/lucene-analysis-common-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-analysis-common-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 1475363327951..0000000000000
--- a/server/licenses/lucene-analysis-common-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5a90d530649b981da658ddda7889f1a3db2eec84
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..773ae4902d257
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+ec67313245fcdb584b98790b6798a59f2f171f39
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-backward-codecs-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index f7ceb099c3bcd..0000000000000
--- a/server/licenses/lucene-backward-codecs-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-077984d39a5ffcdf1d3b6a41f0bc91352ad1a9a6
\ No newline at end of file
diff --git a/server/licenses/lucene-core-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..6815a521b7287
--- /dev/null
+++ b/server/licenses/lucene-core-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+6f5d5f4968f6e2f066a532d0c7a8ed0fbb9aec7d
\ No newline at end of file
diff --git a/server/licenses/lucene-core-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-core-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index d4cc15a3c336f..0000000000000
--- a/server/licenses/lucene-core-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9c2860726c0ce9a32b6c90976e2338e70c9ebfd2
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..08e04201d60ea
--- /dev/null
+++ b/server/licenses/lucene-grouping-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+2992847af15ee3b61b543bddd0e501f3fe178c6f
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-grouping-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 133859fb80755..0000000000000
--- a/server/licenses/lucene-grouping-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d97143f3691c72d62ca591df230399addccc93e7
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..dffe5d7260c5a
--- /dev/null
+++ b/server/licenses/lucene-highlighter-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+530f39370fc0c7b6bc837d3232204670ac62c8e1
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-highlighter-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index aa29f011018c0..0000000000000
--- a/server/licenses/lucene-highlighter-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-22b9996fab3f8cc167c3aa0015f859efc5dd5de3
\ No newline at end of file
diff --git a/server/licenses/lucene-join-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..9f21794438d0b
--- /dev/null
+++ b/server/licenses/lucene-join-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+7368570d058183ccc3d1c5e1d62750ade07162da
\ No newline at end of file
diff --git a/server/licenses/lucene-join-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-join-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 1d1bed7d380bf..0000000000000
--- a/server/licenses/lucene-join-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b1b1b6efd48c353ea930cc159ed526821ae80551
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..6b660dbcc30bb
--- /dev/null
+++ b/server/licenses/lucene-memory-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+148d1bb8a7ff899d082a03e3fab3004e270603c1
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-memory-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 0382d46d7faf4..0000000000000
--- a/server/licenses/lucene-memory-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ab2cb78c651898e51060d0858ce96cc390cbc951
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..f76923a9ce965
--- /dev/null
+++ b/server/licenses/lucene-misc-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+920bc15625ec49f1656cde591a071f4798974158
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-misc-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 26ae55e13363d..0000000000000
--- a/server/licenses/lucene-misc-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f94f74e9f491fb2d3bed9f71ac39d9f7047731d5
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..9fb827c8b5c01
--- /dev/null
+++ b/server/licenses/lucene-queries-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+61561472420037e0cead34ac14cc93ec948ddca5
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-queries-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 839fdc438bfb7..0000000000000
--- a/server/licenses/lucene-queries-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a7fe8b44115414a6514020174b7d75c274492100
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..ae5c8276a7e67
--- /dev/null
+++ b/server/licenses/lucene-queryparser-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+e90bbf2465f7f7fc1c606076d17a69bf7b455439
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-queryparser-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index b95ccbc6c9523..0000000000000
--- a/server/licenses/lucene-queryparser-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-198eed372135943e6971ba78682208944a9386ae
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..c6ba51b1055c3
--- /dev/null
+++ b/server/licenses/lucene-sandbox-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+e67b37e3d5e6fa59eef529f5a24598958006de7e
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-sandbox-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index e33b961389be8..0000000000000
--- a/server/licenses/lucene-sandbox-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-95798905cae9f4f60b4a6f8d8c3e18e59b9e89a8
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..d4009aa5f5e24
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+062e6ecea21b4281d720910e486ed04012ae8b18
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-spatial3d-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 37ad81feecd1f..0000000000000
--- a/server/licenses/lucene-spatial3d-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7e3c6993f306f94618a54ae1afa7cfe75ee7c4ee
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..eb5311d0b1856
--- /dev/null
+++ b/server/licenses/lucene-suggest-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+ade0398fa44d77e59842e40f76ec2910be1f5492
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/server/licenses/lucene-suggest-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index 08e578d7f1186..0000000000000
--- a/server/licenses/lucene-suggest-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1a0f9ecd3b487f93fad3d9b6817e9865b49e01eb
\ No newline at end of file
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index 3c371e59d2a90..e35e1f2ee9afe 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -91,7 +91,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_7_15_0 = new Version(7150099, org.apache.lucene.util.Version.LUCENE_8_9_0);
     public static final Version V_7_15_1 = new Version(7150199, org.apache.lucene.util.Version.LUCENE_8_9_0);
     public static final Version V_7_15_2 = new Version(7150299, org.apache.lucene.util.Version.LUCENE_8_9_0);
-    public static final Version V_7_16_0 = new Version(7160099, org.apache.lucene.util.Version.LUCENE_8_10_0);
+    public static final Version V_7_16_0 = new Version(7160099, org.apache.lucene.util.Version.LUCENE_8_10_1);
     public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_9_0_0);
     public static final Version CURRENT = V_8_0_0;
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java
index 0d281ba4b6944..56b068fa2101d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java
@@ -8,17 +8,20 @@
 
 package org.elasticsearch.action.admin.cluster.migration;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 /**
@@ -57,7 +60,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.value(featureUpgradeStatus);
         }
         builder.endArray();
-        builder.field("upgrade_status", upgradeStatus);
+        builder.field("migration_status", upgradeStatus);
         builder.endObject();
         return builder;
     }
@@ -98,9 +101,18 @@ public String toString() {
     }
 
     public enum UpgradeStatus {
-        UPGRADE_NEEDED,
-        NO_UPGRADE_NEEDED,
-        IN_PROGRESS
+        NO_MIGRATION_NEEDED,
+        MIGRATION_NEEDED,
+        IN_PROGRESS,
+        ERROR;
+
+        public static UpgradeStatus combine(UpgradeStatus... statuses) {
+            int statusOrd = 0;
+            for (UpgradeStatus status : statuses) {
+                statusOrd = Math.max(status.ordinal(), statusOrd);
+            }
+            return UpgradeStatus.values()[statusOrd];
+        }
     }
 
     /**
@@ -111,20 +123,20 @@ public static class FeatureUpgradeStatus implements Writeable, ToXContentObject
         private final String featureName;
         private final Version minimumIndexVersion;
         private final UpgradeStatus upgradeStatus;
-        private final List<IndexVersion> indexVersions;
+        private final List<IndexInfo> indexInfos;
 
         /**
          * @param featureName Name of the feature
          * @param minimumIndexVersion Earliest Elasticsearch version used to create a system index for this feature
          * @param upgradeStatus Whether the feature needs to be upgraded
-         * @param indexVersions A list of this feature's concrete indices and the Elasticsearch version that created them
+         * @param indexInfos A list of this feature's concrete indices and the Elasticsearch version that created them
          */
         public FeatureUpgradeStatus(String featureName, Version minimumIndexVersion,
-                                    UpgradeStatus upgradeStatus, List<IndexVersion> indexVersions) {
+                                    UpgradeStatus upgradeStatus, List<IndexInfo> indexInfos) {
             this.featureName = featureName;
             this.minimumIndexVersion = minimumIndexVersion;
             this.upgradeStatus = upgradeStatus;
-            this.indexVersions = indexVersions;
+            this.indexInfos = indexInfos;
         }
 
         /**
@@ -135,7 +147,7 @@ public FeatureUpgradeStatus(StreamInput in) throws IOException {
             this.featureName = in.readString();
             this.minimumIndexVersion = Version.readVersion(in);
             this.upgradeStatus = in.readEnum(UpgradeStatus.class);
-            this.indexVersions = in.readList(IndexVersion::new);
+            this.indexInfos = in.readList(IndexInfo::new);
         }
 
         public String getFeatureName() {
@@ -150,8 +162,8 @@ public UpgradeStatus getUpgradeStatus() {
             return this.upgradeStatus;
         }
 
-        public List<IndexVersion> getIndexVersions() {
-            return this.indexVersions;
+        public List<IndexInfo> getIndexVersions() {
+            return this.indexInfos;
         }
 
         @Override
@@ -159,7 +171,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(this.featureName);
             Version.writeVersion(this.minimumIndexVersion, out);
             out.writeEnum(this.upgradeStatus);
-            out.writeList(this.indexVersions);
+            out.writeList(this.indexInfos);
         }
 
         @Override
@@ -167,9 +179,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.startObject();
             builder.field("feature_name", this.featureName);
             builder.field("minimum_index_version", this.minimumIndexVersion.toString());
-            builder.field("upgrade_status", this.upgradeStatus);
+            builder.field("migration_status", this.upgradeStatus);
             builder.startArray("indices");
-            for (IndexVersion version : this.indexVersions) {
+            for (IndexInfo version : this.indexInfos) {
                 builder.value(version);
             }
             builder.endArray();
@@ -185,12 +197,12 @@ public boolean equals(Object o) {
             return Objects.equals(featureName, that.featureName)
                 && Objects.equals(minimumIndexVersion, that.minimumIndexVersion)
                 && Objects.equals(upgradeStatus, that.upgradeStatus)
-                && Objects.equals(indexVersions, that.indexVersions);
+                && Objects.equals(indexInfos, that.indexInfos);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(featureName, minimumIndexVersion, upgradeStatus, indexVersions);
+            return Objects.hash(featureName, minimumIndexVersion, upgradeStatus, indexInfos);
         }
 
         @Override
@@ -199,7 +211,7 @@ public String toString() {
                 "featureName='" + featureName + '\'' +
                 ", minimumIndexVersion='" + minimumIndexVersion + '\'' +
                 ", upgradeStatus='" + upgradeStatus + '\'' +
-                ", indexVersions=" + indexVersions +
+                ", indexInfos=" + indexInfos +
                 '}';
         }
     }
@@ -207,26 +219,38 @@ public String toString() {
     /**
      * A data class that holds an index name and the version of Elasticsearch with which that index was created
      */
-    public static class IndexVersion implements Writeable, ToXContentObject {
+    public static class IndexInfo implements Writeable, ToXContentObject {
+        private static final Map<String, String> STACK_TRACE_ENABLED_PARAMS =
+            Map.of(ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE, "false");
+
         private final String indexName;
         private final Version version;
+        @Nullable private final Exception exception; // Present if this index failed
 
         /**
          * @param indexName Name of the index
          * @param version Version of Elasticsearch that created the index
+         * @param exception The exception that this index's migration failed with, if applicable
          */
-        public IndexVersion(String indexName, Version version) {
+        public IndexInfo(String indexName, Version version, Exception exception) {
             this.indexName = indexName;
             this.version = version;
+            this.exception = exception;
         }
 
         /**
          * @param in A stream input for a serialized index version object
          * @throws IOException if we can't deserialize the object
          */
-        public IndexVersion(StreamInput in) throws IOException {
+        public IndexInfo(StreamInput in) throws IOException {
             this.indexName = in.readString();
             this.version = Version.readVersion(in);
+            boolean hasException = in.readBoolean();
+            if (hasException) {
+                this.exception = in.readException();
+            } else {
+                this.exception = null;
+            }
         }
 
         public String getIndexName() {
@@ -237,17 +261,36 @@ public Version getVersion() {
             return this.version;
         }
 
+        public Exception getException() {
+            return this.exception;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(this.indexName);
             Version.writeVersion(this.version, out);
+            if (exception != null) {
+                out.writeBoolean(true);
+                out.writeException(this.exception);
+            } else {
+                out.writeBoolean(false);
+            }
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            Params exceptionParams = new DelegatingMapParams(STACK_TRACE_ENABLED_PARAMS, params);
+
             builder.startObject();
             builder.field("index", this.indexName);
             builder.field("version", this.version.toString());
+            if (exception != null) {
+                builder.startObject("failure_cause");
+                {
+                    ElasticsearchException.generateFailureXContent(builder, exceptionParams, exception, true);
+                }
+                builder.endObject();
+            }
             builder.endObject();
             return builder;
         }
@@ -256,7 +299,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         public boolean equals(Object o) {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
-            IndexVersion that = (IndexVersion) o;
+            IndexInfo that = (IndexInfo) o;
             return indexName.equals(that.indexName) && version.equals(that.version);
         }
 
@@ -267,9 +310,10 @@ public int hashCode() {
 
         @Override
         public String toString() {
-            return "IndexVersion{" +
+            return "IndexInfo{" +
                 "indexName='" + indexName + '\'' +
                 ", version='" + version + '\'' +
+                ", exception='" + exception.getMessage() + "'" +
                 '}';
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java
index b8777d91437e1..04b1031cd6dc7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java
@@ -19,25 +19,38 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.upgrades.FeatureMigrationResults;
+import org.elasticsearch.upgrades.SingleFeatureMigrationResult;
+import org.elasticsearch.upgrades.SystemIndexMigrationTaskState;
 
 import java.util.Collection;
+import java.util.Comparator;
 import java.util.List;
-import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_UPGRADE_NEEDED;
-import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.UPGRADE_NEEDED;
+import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR;
+import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS;
+import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED;
+import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED;
+import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME;
 
 /**
  * Transport class for the get feature upgrade status action
  */
 public class TransportGetFeatureUpgradeStatusAction extends TransportMasterNodeAction<
-        GetFeatureUpgradeStatusRequest,
-        GetFeatureUpgradeStatusResponse> {
+    GetFeatureUpgradeStatusRequest,
+    GetFeatureUpgradeStatusResponse> {
+
+    /**
+     * This version is only valid for >=8.0.0 and should be changed on backport.
+     */
+    public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_8_0_0;
 
     private final SystemIndices systemIndices;
 
@@ -65,55 +78,91 @@ public TransportGetFeatureUpgradeStatusAction(
     }
 
     @Override
-    protected void masterOperation(Task task, GetFeatureUpgradeStatusRequest request, ClusterState state,
-                                   ActionListener<GetFeatureUpgradeStatusResponse> listener) throws Exception {
-
-        List<GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus> features = systemIndices.getFeatures().entrySet().stream()
-            .sorted(Map.Entry.comparingByKey())
-            .map(entry -> getFeatureUpgradeStatus(state, entry))
+    protected void masterOperation(
+        Task task,
+        GetFeatureUpgradeStatusRequest request,
+        ClusterState state,
+        ActionListener<GetFeatureUpgradeStatusResponse> listener
+    ) throws Exception {
+
+        List<GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus> features = systemIndices.getFeatures()
+            .values()
+            .stream()
+            .sorted(Comparator.comparing(SystemIndices.Feature::getName))
+            .map(feature -> getFeatureUpgradeStatus(state, feature))
             .collect(Collectors.toList());
 
-        boolean isUpgradeNeeded = features.stream()
-                .map(GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus::getMinimumIndexVersion)
-                .min(Version::compareTo)
-                .orElse(Version.CURRENT)
-                .before(Version.V_7_0_0);
+        GetFeatureUpgradeStatusResponse.UpgradeStatus status = features.stream()
+            .map(GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus::getUpgradeStatus)
+            .reduce(GetFeatureUpgradeStatusResponse.UpgradeStatus::combine)
+            .orElseGet(() -> {
+                assert false : "get feature statuses API doesn't have any features";
+                return NO_MIGRATION_NEEDED;
+            });
 
-        listener.onResponse(new GetFeatureUpgradeStatusResponse(features, isUpgradeNeeded ? UPGRADE_NEEDED : NO_UPGRADE_NEEDED));
+        listener.onResponse(new GetFeatureUpgradeStatusResponse(features, status));
     }
 
-    // visible for testing
     static GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus getFeatureUpgradeStatus(
-        ClusterState state, Map.Entry<String, SystemIndices.Feature> entry) {
+        ClusterState state,
+        SystemIndices.Feature feature
+    ) {
+        String featureName = feature.getName();
 
-        String featureName = entry.getKey();
-        SystemIndices.Feature feature = entry.getValue();
+        final String currentFeature = Optional.ofNullable(
+            state.metadata().<PersistentTasksCustomMetadata>custom(PersistentTasksCustomMetadata.TYPE)
+        )
+            .map(tasksMetdata -> tasksMetdata.getTask(SYSTEM_INDEX_UPGRADE_TASK_NAME))
+            .map(task -> task.getState())
+            .map(taskState -> ((SystemIndexMigrationTaskState) taskState).getCurrentFeature())
+            .orElse(null);
 
-        List<GetFeatureUpgradeStatusResponse.IndexVersion> indexVersions = getIndexVersions(state, feature);
+        List<GetFeatureUpgradeStatusResponse.IndexInfo> indexInfos = getIndexInfos(state, feature);
 
-        Version minimumVersion = indexVersions.stream()
-            .map(GetFeatureUpgradeStatusResponse.IndexVersion::getVersion)
+        Version minimumVersion = indexInfos.stream()
+            .map(GetFeatureUpgradeStatusResponse.IndexInfo::getVersion)
             .min(Version::compareTo)
             .orElse(Version.CURRENT);
-
-        return new GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus(
-            featureName,
-            minimumVersion,
-            minimumVersion.before(Version.V_7_0_0) ? UPGRADE_NEEDED : NO_UPGRADE_NEEDED,
-            indexVersions
-        );
+        GetFeatureUpgradeStatusResponse.UpgradeStatus initialStatus;
+        if (featureName.equals(currentFeature)) {
+            initialStatus = IN_PROGRESS;
+        } else if (minimumVersion.before(NO_UPGRADE_REQUIRED_VERSION)) {
+            initialStatus = MIGRATION_NEEDED;
+        } else {
+            initialStatus = NO_MIGRATION_NEEDED;
+        }
+
+        GetFeatureUpgradeStatusResponse.UpgradeStatus status = indexInfos.stream()
+            .filter(idxInfo -> idxInfo.getException() != null)
+            .findFirst()
+            .map(idxInfo -> ERROR)
+            .map(idxStatus -> GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(idxStatus, initialStatus))
+            .orElse(initialStatus);
+
+        return new GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus(featureName, minimumVersion, status, indexInfos);
     }
 
     // visible for testing
-    static List<GetFeatureUpgradeStatusResponse.IndexVersion> getIndexVersions(ClusterState state, SystemIndices.Feature feature) {
+    static List<GetFeatureUpgradeStatusResponse.IndexInfo> getIndexInfos(ClusterState state, SystemIndices.Feature feature) {
+        final SingleFeatureMigrationResult featureStatus = Optional.ofNullable(
+            (FeatureMigrationResults) state.metadata().custom(FeatureMigrationResults.TYPE)
+        ).map(FeatureMigrationResults::getFeatureStatuses).map(results -> results.get(feature.getName())).orElse(null);
+
+        final String failedFeatureName = featureStatus == null ? null : featureStatus.getFailedIndexName();
+        final Exception exception = featureStatus == null ? null : featureStatus.getException();
+
         return Stream.of(feature.getIndexDescriptors(), feature.getAssociatedIndexDescriptors())
             .flatMap(Collection::stream)
             .flatMap(descriptor -> descriptor.getMatchingIndices(state.metadata()).stream())
             .sorted(String::compareTo)
             .map(index -> state.metadata().index(index))
-            .map(indexMetadata -> new GetFeatureUpgradeStatusResponse.IndexVersion(
-                indexMetadata.getIndex().getName(),
-                indexMetadata.getCreationVersion()))
+            .map(
+                indexMetadata -> new GetFeatureUpgradeStatusResponse.IndexInfo(
+                    indexMetadata.getIndex().getName(),
+                    indexMetadata.getCreationVersion(),
+                    indexMetadata.getIndex().getName().equals(failedFeatureName) ? exception : null
+                )
+            )
             .collect(Collectors.toList());
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java
index 3ede3d4c4b463..b7ea3d4c7f817 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java
@@ -8,6 +8,9 @@
 
 package org.elasticsearch.action.admin.cluster.migration;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
@@ -18,12 +21,18 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.upgrades.SystemIndexMigrationTaskParams;
 
-import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction.getFeatureUpgradeStatus;
+import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME;
 
 /**
  * Transport action for post feature upgrade action
@@ -31,8 +40,10 @@
 public class TransportPostFeatureUpgradeAction extends TransportMasterNodeAction<
     PostFeatureUpgradeRequest,
     PostFeatureUpgradeResponse> {
+    private static final Logger logger = LogManager.getLogger(TransportPostFeatureUpgradeAction.class);
 
     private final SystemIndices systemIndices;
+    private final PersistentTasksService persistentTasksService;
 
     @Inject
     public TransportPostFeatureUpgradeAction(
@@ -41,7 +52,8 @@ public TransportPostFeatureUpgradeAction(
         ActionFilters actionFilters,
         ClusterService clusterService,
         IndexNameExpressionResolver indexNameExpressionResolver,
-        SystemIndices systemIndices
+        SystemIndices systemIndices,
+        PersistentTasksService persistentTasksService
     ) {
         super(
             PostFeatureUpgradeAction.NAME,
@@ -55,20 +67,46 @@ public TransportPostFeatureUpgradeAction(
             ThreadPool.Names.SAME
         );
         this.systemIndices = systemIndices;
+        this.persistentTasksService = persistentTasksService;
     }
 
     @Override
-    protected void masterOperation(Task task, PostFeatureUpgradeRequest request, ClusterState state,
-                                   ActionListener<PostFeatureUpgradeResponse> listener) throws Exception {
-        List<PostFeatureUpgradeResponse.Feature> features = new ArrayList<>();
-        features.add(new PostFeatureUpgradeResponse.Feature("security"));
-        listener.onResponse(new PostFeatureUpgradeResponse(
-            // TODO: implement operation for this action
-                true, features, null, null));
+    protected void masterOperation(
+        Task task,
+        PostFeatureUpgradeRequest request,
+        ClusterState state,
+        ActionListener<PostFeatureUpgradeResponse> listener
+    ) throws Exception {
+        List<PostFeatureUpgradeResponse.Feature> featuresToMigrate = systemIndices.getFeatures()
+            .values()
+            .stream()
+            .map(feature -> getFeatureUpgradeStatus(state, feature))
+            .filter(status -> status.getUpgradeStatus().equals(GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED))
+            .map(GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus::getFeatureName)
+            .map(PostFeatureUpgradeResponse.Feature::new)
+            .sorted(Comparator.comparing(PostFeatureUpgradeResponse.Feature::getFeatureName)) // consistent ordering to simplify testing
+            .collect(Collectors.toList());
+
+        if (featuresToMigrate.isEmpty() == false) {
+            persistentTasksService.sendStartRequest(
+                SYSTEM_INDEX_UPGRADE_TASK_NAME,
+                SYSTEM_INDEX_UPGRADE_TASK_NAME,
+                new SystemIndexMigrationTaskParams(),
+                ActionListener.wrap(startedTask -> {
+                    listener.onResponse(new PostFeatureUpgradeResponse(true, featuresToMigrate, null, null));
+                }, ex -> {
+                    logger.error("failed to start system index upgrade task", ex);
+
+                    listener.onResponse(new PostFeatureUpgradeResponse(false, null, null, new ElasticsearchException(ex)));
+                })
+            );
+        } else {
+            listener.onResponse(new PostFeatureUpgradeResponse(false, null, "No system indices require migration", null));
+        }
     }
 
     @Override
     protected ClusterBlockException checkBlock(PostFeatureUpgradeRequest request, ClusterState state) {
-        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
index 2be13918c797e..0e013dbe93d14 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.node.hotthreads;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -22,6 +23,7 @@ public class NodesHotThreadsRequest extends BaseNodesRequest<NodesHotThreadsRequ
 
     int threads = 3;
     HotThreads.ReportType type = HotThreads.ReportType.CPU;
+    HotThreads.SortOrder sortOrder = HotThreads.SortOrder.TOTAL;
     TimeValue interval = new TimeValue(500, TimeUnit.MILLISECONDS);
     int snapshots = 10;
     boolean ignoreIdleThreads = true;
@@ -34,6 +36,9 @@ public NodesHotThreadsRequest(StreamInput in) throws IOException {
         type = HotThreads.ReportType.of(in.readString());
         interval = in.readTimeValue();
         snapshots = in.readInt();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            sortOrder = HotThreads.SortOrder.of(in.readString());
+        }
     }
 
     /**
@@ -78,6 +83,15 @@ public HotThreads.ReportType type() {
         return this.type;
     }
 
+    public NodesHotThreadsRequest sortOrder(HotThreads.SortOrder sortOrder) {
+        this.sortOrder = sortOrder;
+        return this;
+    }
+
+    public HotThreads.SortOrder sortOrder() {
+        return this.sortOrder;
+    }
+
     public NodesHotThreadsRequest interval(TimeValue interval) {
         this.interval = interval;
         return this;
@@ -104,5 +118,8 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(type.getTypeValue());
         out.writeTimeValue(interval);
         out.writeInt(snapshots);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeString(sortOrder.getOrderValue());
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java
index 50751c7e1f9c3..03f46d3abba6e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java
@@ -35,6 +35,11 @@ public NodesHotThreadsRequestBuilder setType(HotThreads.ReportType type) {
         return this;
     }
 
+    public NodesHotThreadsRequestBuilder setSortOrder(HotThreads.SortOrder sortOrder) {
+        request.sortOrder(sortOrder);
+        return this;
+    }
+
     public NodesHotThreadsRequestBuilder setInterval(TimeValue interval) {
         request.interval(interval);
         return this;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
index fb37e9497f507..437a3383eea85 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
@@ -59,6 +59,7 @@ protected NodeHotThreads nodeOperation(NodeRequest request, Task task) {
         HotThreads hotThreads = new HotThreads()
                 .busiestThreads(request.request.threads)
                 .type(request.request.type)
+                .sortOrder(request.request.sortOrder)
                 .interval(request.request.interval)
                 .threadElementsSnapshotCount(request.request.snapshots)
                 .ignoreIdleThreads(request.request.ignoreIdleThreads);
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index de226227365b9..2a86664f6a982 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.jdk.JarHell;
+import org.elasticsearch.monitor.jvm.HotThreads;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.monitor.os.OsProbe;
 import org.elasticsearch.monitor.process.ProcessProbe;
@@ -156,6 +157,7 @@ static void initializeProbes() {
         ProcessProbe.getInstance();
         OsProbe.getInstance();
         JvmInfo.jvmInfo();
+        HotThreads.initializeRuntimeMonitoring();
     }
 
     private void setup(boolean addShutdownHook, Environment environment) throws BootstrapException {
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index e7bc6e4f7ce42..302a0e02bcd71 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.MetadataMappingService;
-import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.routing.DelayedAllocationService;
@@ -50,7 +49,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
-
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
@@ -60,7 +58,6 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.ingest.IngestMetadata;
@@ -71,8 +68,9 @@
 import org.elasticsearch.snapshots.SnapshotsInfoService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskResultsService;
-import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.upgrades.FeatureMigrationResults;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.ParseField;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -138,6 +136,12 @@ public static List<Entry> getNamedWriteables() {
             ComposableIndexTemplateMetadata::readDiffFrom);
         registerMetadataCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom);
         registerMetadataCustom(entries, NodesShutdownMetadata.TYPE, NodesShutdownMetadata::new, NodesShutdownMetadata::readDiffFrom);
+        registerMetadataCustom(
+            entries,
+            FeatureMigrationResults.TYPE,
+            FeatureMigrationResults::new,
+            FeatureMigrationResults::readDiffFrom
+        );
 
         // Task Status (not Diffable)
         entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
@@ -266,7 +270,6 @@ protected void configure() {
         bind(MetadataIndexStateService.class).asEagerSingleton();
         bind(MetadataMappingService.class).asEagerSingleton();
         bind(MetadataIndexAliasesService.class).asEagerSingleton();
-        bind(MetadataUpdateSettingsService.class).asEagerSingleton();
         bind(MetadataIndexTemplateService.class).asEagerSingleton();
         bind(IndexNameExpressionResolver.class).toInstance(indexNameExpressionResolver);
         bind(DelayedAllocationService.class).asEagerSingleton();
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
index f4041e76218ca..21a965f3fc60f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
@@ -9,6 +9,7 @@
 
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.LockObtainFailedException;
@@ -22,20 +23,22 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.metadata.ComponentTemplateMetadata;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.DataStreamMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.NodeMetadata;
 import org.elasticsearch.gateway.PersistedClusterStateService;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -68,7 +71,8 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
         public <T, C> T parseNamedObject(Class<T> categoryClass, String name, XContentParser parser, C context) throws IOException {
             // Currently, two unknown top-level objects are present
             if (Metadata.Custom.class.isAssignableFrom(categoryClass)) {
-                if (DataStreamMetadata.TYPE.equals(name)) {
+                if (DataStreamMetadata.TYPE.equals(name) || ComposableIndexTemplateMetadata.TYPE.equals(name)
+                    || ComponentTemplateMetadata.TYPE.equals(name)) {
                     // DataStreamMetadata is used inside Metadata class for validation purposes and building the indicesLookup,
                     // therefor even es node commands need to be able to parse it.
                     return super.parseNamedObject(categoryClass, name, parser, context);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
index 2548431276020..ecfece366f7ad 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java
@@ -129,7 +129,7 @@ public String toString() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(TEMPLATE.getPreferredName(), this.template);
+        builder.field(TEMPLATE.getPreferredName(), this.template, params);
         if (this.version != null) {
             builder.field(VERSION.getPreferredName(), this.version);
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
index c655145bd7ff0..883372b9a4638 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
@@ -98,7 +98,7 @@ public static ComponentTemplateMetadata fromXContent(XContentParser parser) thro
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(COMPONENT_TEMPLATE.getPreferredName());
         for (Map.Entry<String, ComponentTemplate> template : componentTemplates.entrySet()) {
-            builder.field(template.getKey(), template.getValue());
+            builder.field(template.getKey(), template.getValue(), params);
         }
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
index 80ab79c2b24ad..79581380d296b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
@@ -202,7 +202,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.stringListField(INDEX_PATTERNS.getPreferredName(), this.indexPatterns);
         if (this.template != null) {
-            builder.field(TEMPLATE.getPreferredName(), this.template);
+            builder.field(TEMPLATE.getPreferredName(), this.template, params);
         }
         if (this.componentTemplates != null) {
             builder.stringListField(COMPOSED_OF.getPreferredName(), this.componentTemplates);
@@ -217,7 +217,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(METADATA.getPreferredName(), metadata);
         }
         if (this.dataStreamTemplate != null) {
-            builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate);
+            builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate, params);
         }
         if (this.allowAutoCreate != null) {
             builder.field(ALLOW_AUTO_CREATE.getPreferredName(), allowAutoCreate);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java
index 67fd84779f6b7..1fe3f3ab8d8c2 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java
@@ -99,7 +99,7 @@ public void writeTo(StreamOutput out) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(INDEX_TEMPLATE.getPreferredName());
         for (Map.Entry<String, ComposableIndexTemplate> template : indexTemplates.entrySet()) {
-            builder.field(template.getKey(), template.getValue());
+            builder.field(template.getKey(), template.getValue(), params);
         }
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
index add6ff97597fc..b7781680aca92 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
@@ -103,7 +103,7 @@ public List<String> resolveIndexAbstractions(Iterable<String> indices, IndicesOp
             } else if (dateMathName.equals(indexAbstraction)) {
                 if (minus) {
                     finalIndices.remove(indexAbstraction);
-                } else {
+                } else if (indicesOptions.ignoreUnavailable() == false || availableIndexAbstractions.contains(indexAbstraction)) {
                     finalIndices.add(indexAbstraction);
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java
index e25808bc1d24f..9800f732e6d3a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -57,7 +56,6 @@ public class MetadataUpdateSettingsService {
     private final ShardLimitValidator shardLimitValidator;
     private final ThreadPool threadPool;
 
-    @Inject
     public MetadataUpdateSettingsService(ClusterService clusterService, AllocationService allocationService,
                                          IndexScopedSettings indexScopedSettings, IndicesService indicesService,
                                          ShardLimitValidator shardLimitValidator, ThreadPool threadPool) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java
index ffb23b6835b7b..8e7833aae6ee1 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -27,6 +28,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 
 import java.io.IOException;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -47,8 +49,18 @@ public class Template extends AbstractDiffable<Template> implements ToXContentOb
 
     static {
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS);
-        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) ->
-            new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(p.mapOrdered()))), MAPPINGS);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
+            XContentParser.Token token = p.currentToken();
+            if (token == XContentParser.Token.VALUE_STRING) {
+                return new CompressedXContent(Base64.getDecoder().decode(p.text()));
+            } else if(token == XContentParser.Token.VALUE_EMBEDDED_OBJECT){
+                return new CompressedXContent(p.binaryValue());
+            } else if (token == XContentParser.Token.START_OBJECT){
+                return new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(p.mapOrdered())));
+            } else {
+                throw new IllegalArgumentException("Unexpected token: " + token);
+            }
+        }, MAPPINGS, ObjectParser.ValueType.VALUE_OBJECT_ARRAY);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
             Map<String, AliasMetadata> aliasMap = new HashMap<>();
             while ((p.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -160,11 +172,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.endObject();
         }
         if (this.mappings != null) {
-            Map<String, Object> uncompressedMapping =
-                XContentHelper.convertToMap(this.mappings.uncompressed(), true, XContentType.JSON).v2();
-            if (uncompressedMapping.size() > 0) {
-                builder.field(MAPPINGS.getPreferredName());
-                builder.map(reduceMapping(uncompressedMapping));
+            String context = params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API);
+            boolean binary = params.paramAsBoolean("binary", false);
+            if (Metadata.CONTEXT_MODE_API.equals(context) || binary == false) {
+                Map<String, Object> uncompressedMapping =
+                    XContentHelper.convertToMap(this.mappings.uncompressed(), true, XContentType.JSON).v2();
+                if (uncompressedMapping.size() > 0) {
+                    builder.field(MAPPINGS.getPreferredName());
+                    builder.map(reduceMapping(uncompressedMapping));
+                }
+            } else {
+                builder.field(MAPPINGS.getPreferredName(), mappings.compressed());
             }
         }
         if (this.aliases != null) {
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java
index 69cf081da0d04..6e3b0f8135918 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java
@@ -28,7 +28,7 @@ public interface LeafFieldData extends Accountable, Releasable {
     /**
      * Returns an {@code Field} for use in accessing field values in scripting.
      */
-    default DocValuesField<?> getScriptField(String name) {
+    default DocValuesField getScriptField(String name) {
         throw new UnsupportedOperationException();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java
index 0be7dca237845..eafe27ad446e6 100644
--- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java
+++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java
@@ -23,13 +23,13 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.core.Booleans;
-import org.elasticsearch.core.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.TriConsumer;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.plugins.SystemIndexPlugin;
 import org.elasticsearch.snapshots.SnapshotsService;
@@ -64,7 +64,8 @@ public class SystemIndices {
     private static final Automaton EMPTY = Automata.makeEmpty();
 
     private static final Map<String, Feature> SERVER_SYSTEM_INDEX_DESCRIPTORS = Map.of(
-        TASKS_FEATURE_NAME, new Feature(TASKS_FEATURE_NAME, "Manages task results", List.of(TASKS_DESCRIPTOR))
+        TASKS_FEATURE_NAME,
+        new Feature(TASKS_FEATURE_NAME, "Manages task results", List.of(TASKS_DESCRIPTOR))
     );
 
     private final Automaton systemNameAutomaton;
@@ -107,7 +108,8 @@ static void ensurePatternsAllowSuffix(Map<String, Feature> features) {
         final List<String> descriptorsWithNoRoomForSuffix = features.entrySet()
             .stream()
             .flatMap(
-                feature -> feature.getValue().getIndexDescriptors()
+                feature -> feature.getValue()
+                    .getIndexDescriptors()
                     .stream()
                     // The below filter & map are inside the enclosing flapMap so we have access to both the feature and the descriptor
                     .filter(descriptor -> overlaps(descriptor.getIndexPattern(), suffixPattern) == false)
@@ -158,31 +160,38 @@ private static Map<String, CharacterRunAutomaton> getProductToSystemIndicesMap(M
         for (Feature feature : descriptors.values()) {
             feature.getIndexDescriptors().forEach(systemIndexDescriptor -> {
                 if (systemIndexDescriptor.isExternal()) {
-                    systemIndexDescriptor.getAllowedElasticProductOrigins().forEach(origin ->
-                        productToSystemIndicesMap.compute(origin, (key, value) -> {
+                    systemIndexDescriptor.getAllowedElasticProductOrigins()
+                        .forEach(origin -> productToSystemIndicesMap.compute(origin, (key, value) -> {
                             Automaton automaton = SystemIndexDescriptor.buildAutomaton(
-                                systemIndexDescriptor.getIndexPattern(), systemIndexDescriptor.getAliasName());
+                                systemIndexDescriptor.getIndexPattern(),
+                                systemIndexDescriptor.getAliasName()
+                            );
                             return value == null ? automaton : Operations.union(value, automaton);
-                        })
-                    );
+                        }));
                 }
             });
             feature.getDataStreamDescriptors().forEach(dataStreamDescriptor -> {
                 if (dataStreamDescriptor.isExternal()) {
-                    dataStreamDescriptor.getAllowedElasticProductOrigins().forEach(origin ->
-                        productToSystemIndicesMap.compute(origin, (key, value) -> {
+                    dataStreamDescriptor.getAllowedElasticProductOrigins()
+                        .forEach(origin -> productToSystemIndicesMap.compute(origin, (key, value) -> {
                             Automaton automaton = SystemIndexDescriptor.buildAutomaton(
-                                dataStreamDescriptor.getBackingIndexPattern(), dataStreamDescriptor.getDataStreamName());
+                                dataStreamDescriptor.getBackingIndexPattern(),
+                                dataStreamDescriptor.getDataStreamName()
+                            );
                             return value == null ? automaton : Operations.union(value, automaton);
-                        })
-                    );
+                        }));
                 }
             });
         }
 
-        return productToSystemIndicesMap.entrySet().stream()
-            .collect(Collectors.toUnmodifiableMap(Entry::getKey, entry ->
-                new CharacterRunAutomaton(MinimizationOperations.minimize(entry.getValue(), Integer.MAX_VALUE))));
+        return productToSystemIndicesMap.entrySet()
+            .stream()
+            .collect(
+                Collectors.toUnmodifiableMap(
+                    Entry::getKey,
+                    entry -> new CharacterRunAutomaton(MinimizationOperations.minimize(entry.getValue(), Integer.MAX_VALUE))
+                )
+            );
     }
 
     /**
@@ -235,10 +244,23 @@ public Automaton getSystemNameAutomaton() {
         return systemNameAutomaton;
     }
 
+    /**
+     * Checks whether an index is a net-new system index, meaning we can apply non-BWC behavior to it.
+     * @param indexName The index name to check.
+     * @return {@code true} if the given index is covered by a net-new system index descriptor, {@code false} otherwise.
+     */
     public boolean isNetNewSystemIndex(String indexName) {
         return netNewSystemIndexAutomaton.run(indexName);
     }
 
+    /**
+     * Used to determine which executor should be used for operations on this index. See {@link ExecutorSelector} docs for
+     * details.
+     */
+    public ExecutorSelector getExecutorSelector() {
+        return executorSelector;
+    }
+
     /**
      * Finds a single matching {@link SystemIndexDescriptor}, if any, for the given index name.
      * @param name the name of the index
@@ -246,7 +268,8 @@ public boolean isNetNewSystemIndex(String indexName) {
      * @throws IllegalStateException if multiple descriptors match the name
      */
     public @Nullable SystemIndexDescriptor findMatchingDescriptor(String name) {
-        final List<SystemIndexDescriptor> matchingDescriptors = featureDescriptors.values().stream()
+        final List<SystemIndexDescriptor> matchingDescriptors = featureDescriptors.values()
+            .stream()
             .flatMap(feature -> feature.getIndexDescriptors().stream())
             .filter(descriptor -> descriptor.matchesIndexPattern(name))
             .collect(toUnmodifiableList());
@@ -257,13 +280,20 @@ public boolean isNetNewSystemIndex(String indexName) {
             return matchingDescriptors.get(0);
         } else {
             // This should be prevented by failing on overlapping patterns at startup time, but is here just in case.
-            StringBuilder errorMessage = new StringBuilder()
-                .append("index name [")
+            StringBuilder errorMessage = new StringBuilder().append("index name [")
                 .append(name)
                 .append("] is claimed as a system index by multiple system index patterns: [")
-                .append(matchingDescriptors.stream()
-                    .map(descriptor -> "pattern: [" + descriptor.getIndexPattern() +
-                        "], description: [" + descriptor.getDescription() + "]").collect(Collectors.joining("; ")));
+                .append(
+                    matchingDescriptors.stream()
+                        .map(
+                            descriptor -> "pattern: ["
+                                + descriptor.getIndexPattern()
+                                + "], description: ["
+                                + descriptor.getDescription()
+                                + "]"
+                        )
+                        .collect(Collectors.joining("; "))
+                );
             // Throw AssertionError if assertions are enabled, or a regular exception otherwise:
             assert false : errorMessage.toString();
             throw new IllegalStateException(errorMessage.toString());
@@ -277,7 +307,8 @@ public boolean isNetNewSystemIndex(String indexName) {
      * @throws IllegalStateException if multiple descriptors match the name
      */
     public @Nullable SystemDataStreamDescriptor findMatchingDataStreamDescriptor(String name) {
-        final List<SystemDataStreamDescriptor> matchingDescriptors = featureDescriptors.values().stream()
+        final List<SystemDataStreamDescriptor> matchingDescriptors = featureDescriptors.values()
+            .stream()
             .flatMap(feature -> feature.getDataStreamDescriptors().stream())
             .filter(descriptor -> descriptor.getDataStreamName().equals(name))
             .collect(toUnmodifiableList());
@@ -288,13 +319,20 @@ public boolean isNetNewSystemIndex(String indexName) {
             return matchingDescriptors.get(0);
         } else {
             // This should be prevented by failing on overlapping patterns at startup time, but is here just in case.
-            StringBuilder errorMessage = new StringBuilder()
-                .append("DataStream name [")
+            StringBuilder errorMessage = new StringBuilder().append("DataStream name [")
                 .append(name)
                 .append("] is claimed as a system data stream by multiple descriptors: [")
-                .append(matchingDescriptors.stream()
-                    .map(descriptor -> "name: [" + descriptor.getDataStreamName() +
-                        "], description: [" + descriptor.getDescription() + "]").collect(Collectors.joining("; ")));
+                .append(
+                    matchingDescriptors.stream()
+                        .map(
+                            descriptor -> "name: ["
+                                + descriptor.getDataStreamName()
+                                + "], description: ["
+                                + descriptor.getDescription()
+                                + "]"
+                        )
+                        .collect(Collectors.joining("; "))
+                );
             // Throw AssertionError if assertions are enabled, or a regular exception otherwise:
             assert false : errorMessage.toString();
             throw new IllegalStateException(errorMessage.toString());
@@ -342,14 +380,13 @@ public Map<String, Feature> getFeatures() {
     }
 
     private static Automaton buildIndexAutomaton(Map<String, Feature> descriptors) {
-        Optional<Automaton> automaton = descriptors.values().stream()
-            .map(SystemIndices::featureToIndexAutomaton)
-            .reduce(Operations::union);
+        Optional<Automaton> automaton = descriptors.values().stream().map(SystemIndices::featureToIndexAutomaton).reduce(Operations::union);
         return MinimizationOperations.minimize(automaton.orElse(EMPTY), Integer.MAX_VALUE);
     }
 
     private static CharacterRunAutomaton buildNetNewIndexCharacterRunAutomaton(Map<String, Feature> featureDescriptors) {
-        Optional<Automaton> automaton = featureDescriptors.values().stream()
+        Optional<Automaton> automaton = featureDescriptors.values()
+            .stream()
             .flatMap(feature -> feature.getIndexDescriptors().stream())
             .filter(SystemIndexDescriptor::isNetNew)
             .map(descriptor -> SystemIndexDescriptor.buildAutomaton(descriptor.getIndexPattern(), descriptor.getAliasName()))
@@ -358,7 +395,8 @@ private static CharacterRunAutomaton buildNetNewIndexCharacterRunAutomaton(Map<S
     }
 
     private static Automaton featureToIndexAutomaton(Feature feature) {
-        Optional<Automaton> systemIndexAutomaton = feature.getIndexDescriptors().stream()
+        Optional<Automaton> systemIndexAutomaton = feature.getIndexDescriptors()
+            .stream()
             .map(descriptor -> SystemIndexDescriptor.buildAutomaton(descriptor.getIndexPattern(), descriptor.getAliasName()))
             .reduce(Operations::union);
 
@@ -366,7 +404,8 @@ private static Automaton featureToIndexAutomaton(Feature feature) {
     }
 
     private static Automaton buildDataStreamAutomaton(Map<String, Feature> descriptors) {
-        Optional<Automaton> automaton = descriptors.values().stream()
+        Optional<Automaton> automaton = descriptors.values()
+            .stream()
             .flatMap(feature -> feature.getDataStreamDescriptors().stream())
             .map(SystemDataStreamDescriptor::getDataStreamName)
             .map(dsName -> SystemIndexDescriptor.buildAutomaton(dsName, null))
@@ -381,25 +420,25 @@ private static Predicate<String> buildDataStreamNamePredicate(Map<String, Featur
     }
 
     private static Automaton buildDataStreamBackingIndicesAutomaton(Map<String, Feature> descriptors) {
-        Optional<Automaton> automaton = descriptors.values().stream()
+        Optional<Automaton> automaton = descriptors.values()
+            .stream()
             .map(SystemIndices::featureToDataStreamBackingIndicesAutomaton)
             .reduce(Operations::union);
         return MinimizationOperations.minimize(automaton.orElse(EMPTY), Integer.MAX_VALUE);
     }
 
     private static Automaton featureToDataStreamBackingIndicesAutomaton(Feature feature) {
-        Optional<Automaton> systemDataStreamAutomaton = feature.getDataStreamDescriptors().stream()
-            .map(descriptor -> SystemIndexDescriptor.buildAutomaton(
-                descriptor.getBackingIndexPattern(),
-                null
-            ))
+        Optional<Automaton> systemDataStreamAutomaton = feature.getDataStreamDescriptors()
+            .stream()
+            .map(descriptor -> SystemIndexDescriptor.buildAutomaton(descriptor.getBackingIndexPattern(), null))
             .reduce(Operations::union);
         return systemDataStreamAutomaton.orElse(EMPTY);
     }
 
     public SystemDataStreamDescriptor validateDataStreamAccess(String dataStreamName, ThreadContext threadContext) {
         if (systemDataStreamPredicate.test(dataStreamName)) {
-            SystemDataStreamDescriptor dataStreamDescriptor = featureDescriptors.values().stream()
+            SystemDataStreamDescriptor dataStreamDescriptor = featureDescriptors.values()
+                .stream()
                 .flatMap(feature -> feature.getDataStreamDescriptors().stream())
                 .filter(descriptor -> descriptor.getDataStreamName().equals(dataStreamName))
                 .findFirst()
@@ -413,7 +452,8 @@ public SystemDataStreamDescriptor validateDataStreamAccess(String dataStreamName
                     if (getProductSystemIndexNamePredicate(threadContext).test(dataStreamName) == false) {
                         throw dataStreamAccessException(
                             threadContext.getHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY),
-                            dataStreamName);
+                            dataStreamName
+                        );
                     } else {
                         return dataStreamDescriptor;
                     }
@@ -439,21 +479,25 @@ public IllegalArgumentException dataStreamAccessException(ThreadContext threadCo
     public IllegalArgumentException netNewSystemIndexAccessException(ThreadContext threadContext, Collection<String> names) {
         final String product = threadContext.getHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY);
         if (product == null) {
-            return new IllegalArgumentException("Indices " + Arrays.toString(names.toArray(Strings.EMPTY_ARRAY)) +
-                " use and access is reserved for system operations");
+            return new IllegalArgumentException(
+                "Indices " + Arrays.toString(names.toArray(Strings.EMPTY_ARRAY)) + " use and access is reserved for system operations"
+            );
         } else {
-            return new IllegalArgumentException("Indices " + Arrays.toString(names.toArray(Strings.EMPTY_ARRAY)) +
-                " use and access is reserved for system operations");
+            return new IllegalArgumentException(
+                "Indices " + Arrays.toString(names.toArray(Strings.EMPTY_ARRAY)) + " use and access is reserved for system operations"
+            );
         }
     }
 
     IllegalArgumentException dataStreamAccessException(@Nullable String product, String... dataStreamNames) {
         if (product == null) {
-            return new IllegalArgumentException("Data stream(s) " + Arrays.toString(dataStreamNames) +
-                " use and access is reserved for system operations");
+            return new IllegalArgumentException(
+                "Data stream(s) " + Arrays.toString(dataStreamNames) + " use and access is reserved for system operations"
+            );
         } else {
-            return new IllegalArgumentException("Data stream(s) " + Arrays.toString(dataStreamNames) + " may not be accessed by product ["
-                + product + "]");
+            return new IllegalArgumentException(
+                "Data stream(s) " + Arrays.toString(dataStreamNames) + " may not be accessed by product [" + product + "]"
+            );
         }
     }
 
@@ -505,16 +549,18 @@ public enum SystemIndexAccessLevel {
      * @throws IllegalStateException Thrown if any of the index patterns overlaps with another.
      */
     static void checkForOverlappingPatterns(Map<String, Feature> sourceToFeature) {
-        List<Tuple<String, SystemIndexDescriptor>> sourceDescriptorPair = sourceToFeature.entrySet().stream()
+        List<Tuple<String, SystemIndexDescriptor>> sourceDescriptorPair = sourceToFeature.entrySet()
+            .stream()
             .flatMap(entry -> entry.getValue().getIndexDescriptors().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor)))
             .sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message
             .collect(Collectors.toUnmodifiableList());
-        List<Tuple<String, SystemDataStreamDescriptor>> sourceDataStreamDescriptorPair = sourceToFeature.entrySet().stream()
+        List<Tuple<String, SystemDataStreamDescriptor>> sourceDataStreamDescriptorPair = sourceToFeature.entrySet()
+            .stream()
             .filter(entry -> entry.getValue().getDataStreamDescriptors().isEmpty() == false)
-            .flatMap(entry ->
-                entry.getValue().getDataStreamDescriptors().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor)))
-            .sorted(
-                Comparator.comparing(d -> d.v1() + ":" + d.v2().getDataStreamName())) // Consistent ordering -> consistent error message
+            .flatMap(
+                entry -> entry.getValue().getDataStreamDescriptors().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor))
+            )
+            .sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getDataStreamName())) // Consistent ordering -> consistent error message
             .collect(Collectors.toUnmodifiableList());
 
         // This is O(n^2) with the number of system index descriptors, and each check is quadratic with the number of states in the
@@ -523,27 +569,41 @@ static void checkForOverlappingPatterns(Map<String, Feature> sourceToFeature) {
         sourceDescriptorPair.forEach(descriptorToCheck -> {
             List<Tuple<String, SystemIndexDescriptor>> descriptorsMatchingThisPattern = sourceDescriptorPair.stream()
                 .filter(d -> descriptorToCheck.v2() != d.v2()) // Exclude the pattern currently being checked
-                .filter(d -> overlaps(descriptorToCheck.v2(), d.v2()) ||
-                    (d.v2().getAliasName() != null && descriptorToCheck.v2().matchesIndexPattern(d.v2().getAliasName())))
+                .filter(
+                    d -> overlaps(descriptorToCheck.v2(), d.v2())
+                        || (d.v2().getAliasName() != null && descriptorToCheck.v2().matchesIndexPattern(d.v2().getAliasName()))
+                )
                 .collect(toUnmodifiableList());
             if (descriptorsMatchingThisPattern.isEmpty() == false) {
-                throw new IllegalStateException("a system index descriptor [" + descriptorToCheck.v2() + "] from [" +
-                    descriptorToCheck.v1() + "] overlaps with other system index descriptors: [" +
-                    descriptorsMatchingThisPattern.stream()
-                        .map(descriptor -> descriptor.v2() + " from [" + descriptor.v1() + "]")
-                        .collect(Collectors.joining(", ")));
+                throw new IllegalStateException(
+                    "a system index descriptor ["
+                        + descriptorToCheck.v2()
+                        + "] from ["
+                        + descriptorToCheck.v1()
+                        + "] overlaps with other system index descriptors: ["
+                        + descriptorsMatchingThisPattern.stream()
+                            .map(descriptor -> descriptor.v2() + " from [" + descriptor.v1() + "]")
+                            .collect(Collectors.joining(", "))
+                );
             }
 
             List<Tuple<String, SystemDataStreamDescriptor>> dataStreamsMatching = sourceDataStreamDescriptorPair.stream()
-                .filter(dsTuple -> descriptorToCheck.v2().matchesIndexPattern(dsTuple.v2().getDataStreamName()) ||
-                    overlaps(descriptorToCheck.v2().getIndexPattern(), dsTuple.v2().getBackingIndexPattern()))
+                .filter(
+                    dsTuple -> descriptorToCheck.v2().matchesIndexPattern(dsTuple.v2().getDataStreamName())
+                        || overlaps(descriptorToCheck.v2().getIndexPattern(), dsTuple.v2().getBackingIndexPattern())
+                )
                 .collect(toUnmodifiableList());
             if (dataStreamsMatching.isEmpty() == false) {
-                throw new IllegalStateException("a system index descriptor [" + descriptorToCheck.v2() + "] from [" +
-                    descriptorToCheck.v1() + "] overlaps with one or more data stream descriptors: [" +
-                    dataStreamsMatching.stream()
-                        .map(descriptor -> descriptor.v2() + " from [" + descriptor.v1() + "]")
-                        .collect(Collectors.joining(", ")));
+                throw new IllegalStateException(
+                    "a system index descriptor ["
+                        + descriptorToCheck.v2()
+                        + "] from ["
+                        + descriptorToCheck.v1()
+                        + "] overlaps with one or more data stream descriptors: ["
+                        + dataStreamsMatching.stream()
+                            .map(descriptor -> descriptor.v2() + " from [" + descriptor.v1() + "]")
+                            .collect(Collectors.joining(", "))
+                );
             }
         });
     }
@@ -564,17 +624,16 @@ private static Map<String, Feature> buildSystemIndexDescriptorMap(Map<String, Fe
         // put the server items last since we expect less of them
         SERVER_SYSTEM_INDEX_DESCRIPTORS.forEach((source, feature) -> {
             if (map.putIfAbsent(source, feature) != null) {
-                throw new IllegalArgumentException("plugin or module attempted to define the same source [" + source +
-                    "] as a built-in system index");
+                throw new IllegalArgumentException(
+                    "plugin or module attempted to define the same source [" + source + "] as a built-in system index"
+                );
             }
         });
         return Map.copyOf(map);
     }
 
     Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
-        return this.featureDescriptors.values().stream()
-            .flatMap(f -> f.getIndexDescriptors().stream())
-            .collect(Collectors.toList());
+        return this.featureDescriptors.values().stream().flatMap(f -> f.getIndexDescriptors().stream()).collect(Collectors.toList());
     }
 
     /**
@@ -584,8 +643,13 @@ Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
      */
     public static void validateFeatureName(String name, String plugin) {
         if (SnapshotsService.NO_FEATURE_STATES_VALUE.equalsIgnoreCase(name)) {
-            throw new IllegalArgumentException("feature name cannot be reserved name [\"" + SnapshotsService.NO_FEATURE_STATES_VALUE +
-                "\"], but was for plugin [" + plugin + "]");
+            throw new IllegalArgumentException(
+                "feature name cannot be reserved name [\""
+                    + SnapshotsService.NO_FEATURE_STATES_VALUE
+                    + "\"], but was for plugin ["
+                    + plugin
+                    + "]"
+            );
         }
     }
 
@@ -593,6 +657,7 @@ public static void validateFeatureName(String name, String plugin) {
      * Class holding a description of a stateful feature.
      */
     public static class Feature {
+        private final String name;
         private final String description;
         private final Collection<SystemIndexDescriptor> indexDescriptors;
         private final Collection<SystemDataStreamDescriptor> dataStreamDescriptors;
@@ -603,6 +668,7 @@ public static class Feature {
 
         /**
          * Construct a Feature with a custom cleanup function
+         * @param name The name of the feature
          * @param description Description of the feature
          * @param indexDescriptors Collection of objects describing system indices for this feature
          * @param dataStreamDescriptors Collection of objects describing system data streams for this feature
@@ -612,6 +678,7 @@ public static class Feature {
          * @param postMigrationFunction A function that will be called after upgrading all of this plugin's system indices
          */
         public Feature(
+            String name,
             String description,
             Collection<SystemIndexDescriptor> indexDescriptors,
             Collection<SystemDataStreamDescriptor> dataStreamDescriptors,
@@ -620,6 +687,7 @@ public Feature(
             MigrationPreparationHandler preMigrationFunction,
             MigrationCompletionHandler postMigrationFunction
         ) {
+            this.name = name;
             this.description = description;
             this.indexDescriptors = indexDescriptors;
             this.dataStreamDescriptors = dataStreamDescriptors;
@@ -637,6 +705,7 @@ public Feature(
          */
         public Feature(String name, String description, Collection<SystemIndexDescriptor> indexDescriptors) {
             this(
+                name,
                 description,
                 indexDescriptors,
                 Collections.emptyList(),
@@ -668,6 +737,7 @@ public Feature(
             Collection<SystemDataStreamDescriptor> dataStreamDescriptors
         ) {
             this(
+                name,
                 description,
                 indexDescriptors,
                 dataStreamDescriptors,
@@ -693,6 +763,7 @@ public Feature(
          */
         public static Feature fromSystemIndexPlugin(SystemIndexPlugin plugin, Settings settings) {
             return new Feature(
+                plugin.getFeatureName(),
                 plugin.getFeatureDescription(),
                 plugin.getSystemIndexDescriptors(settings),
                 plugin.getSystemDataStreamDescriptors(),
@@ -723,6 +794,10 @@ public TriConsumer<ClusterService, Client, ActionListener<ResetFeatureStateStatu
             return cleanUpFunction;
         }
 
+        public String getName() {
+            return name;
+        }
+
         public MigrationPreparationHandler getPreMigrationFunction() {
             return preMigrationFunction;
         }
@@ -795,13 +870,21 @@ private static void noopPostMigrationFunction(
             listener.onResponse(true);
         }
 
+        /**
+         * Type for the handler that's invoked prior to migrating a Feature's system indices.
+         * See {@link SystemIndexPlugin#prepareForIndicesMigration(ClusterService, Client, ActionListener)}.
+         */
         @FunctionalInterface
-        interface MigrationPreparationHandler {
+        public interface MigrationPreparationHandler {
             void prepareForIndicesMigration(ClusterService clusterService, Client client, ActionListener<Map<String, Object>> listener);
         }
 
+        /**
+         * Type for the handler that's invoked when all of a feature's system indices have been migrated.
+         * See {@link SystemIndexPlugin#indicesMigrationComplete(Map, ClusterService, Client, ActionListener)}.
+         */
         @FunctionalInterface
-        interface MigrationCompletionHandler {
+        public interface MigrationCompletionHandler {
             void indicesMigrationComplete(
                 Map<String, Object> preUpgradeMetadata,
                 ClusterService clusterService,
@@ -809,10 +892,18 @@ void indicesMigrationComplete(
                 ActionListener<Boolean> listener
             );
         }
-    }
 
-    public ExecutorSelector getExecutorSelector() {
-        return executorSelector;
+        public static Feature pluginToFeature(SystemIndexPlugin plugin, Settings settings) {
+            return new Feature(
+                plugin.getFeatureName(),
+                plugin.getFeatureDescription(),
+                plugin.getSystemIndexDescriptors(settings),
+                plugin.getSystemDataStreamDescriptors(),
+                plugin.getAssociatedIndexDescriptors(),
+                plugin::cleanUpFeature,
+                plugin::prepareForIndicesMigration,
+                plugin::indicesMigrationComplete
+            );
+        }
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java b/server/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java
index faa4d3d87ff1d..8f1e47c2e86ca 100644
--- a/server/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java
+++ b/server/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.monitor.jvm;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.time.DateFormatter;
@@ -17,8 +18,6 @@
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
 import java.lang.management.ThreadMXBean;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
 import java.time.Clock;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
@@ -44,6 +43,8 @@ public class HotThreads {
     private TimeValue threadElementsSnapshotDelay = new TimeValue(10, TimeUnit.MILLISECONDS);
     private int threadElementsSnapshotCount = 10;
     private ReportType type = ReportType.CPU;
+    private SortOrder sortOrder = SortOrder.TOTAL;
+
     private boolean ignoreIdleThreads = true;
 
     private static final List<String[]> knownIdleStackFrames = Arrays.asList(
@@ -57,7 +58,7 @@ public class HotThreads {
 
     // NOTE: these are JVM dependent and JVM version dependent
     private static final List<String> knownJDKInternalThreads = Arrays.asList(
-        "Signal Dispatcher", "Finalizer", "Reference Handler", "Notification Thread", "Common-Cleaner", "process reaper"
+        "Signal Dispatcher", "Finalizer", "Reference Handler", "Notification Thread", "Common-Cleaner", "process reaper", "DestroyJavaVM"
     );
 
     public enum ReportType {
@@ -77,6 +78,8 @@ public String getTypeValue() {
             return type;
         }
 
+        // Custom enum from string because of legacy support. The standard Enum.valueOf is static
+        // and cannot be overriden to show a nice error message in case the enum value isn't found
         public static ReportType of(String type) {
             for (var report : values()) {
                 if (report.type.equals(type)) {
@@ -87,6 +90,33 @@ public static ReportType of(String type) {
         }
     }
 
+    public enum SortOrder {
+
+        TOTAL("total"),
+        CPU("cpu");
+
+        private final String order;
+
+        SortOrder(String order) {
+            this.order = order;
+        }
+
+        public String getOrderValue() {
+            return order;
+        }
+
+        // Custom enum from string because of legacy support. The standard Enum.valueOf is static
+        // and cannot be overriden to show a nice error message in case the enum value isn't found
+        public static SortOrder of(String order) {
+            for (var sortOrder : values()) {
+                if (sortOrder.order.equals(order)) {
+                    return sortOrder;
+                }
+            }
+            throw new IllegalArgumentException("sort order not supported [" + order + "]");
+        }
+    }
+
     public HotThreads interval(TimeValue interval) {
         this.interval = interval;
         return this;
@@ -117,9 +147,21 @@ public HotThreads type(ReportType type) {
         return this;
     }
 
+    public HotThreads sortOrder(SortOrder order) {
+        this.sortOrder = order;
+        return this;
+    }
+
     public String detect() throws Exception {
         synchronized (mutex) {
-            return innerDetect(ManagementFactory.getThreadMXBean(), SunThreadInfo.INSTANCE, Thread.currentThread().getId());
+            return innerDetect(
+                ManagementFactory.getThreadMXBean(),
+                SunThreadInfo.INSTANCE,
+                Thread.currentThread().getId(),
+                (interval) -> {
+                    Thread.sleep(interval);
+                    return null;
+                });
         }
     }
 
@@ -161,7 +203,7 @@ Map<Long, ThreadTimeAccumulator> getAllValidThreadInfos(ThreadMXBean threadBean,
                 continue;
             }
             long allocatedBytes = type == ReportType.MEM ? sunThreadInfo.getThreadAllocatedBytes(threadIds[i]) : 0;
-            result.put(threadIds[i], new ThreadTimeAccumulator(threadInfos[i], cpuTime, allocatedBytes));
+            result.put(threadIds[i], new ThreadTimeAccumulator(threadInfos[i], interval, cpuTime, allocatedBytes));
         }
 
         return result;
@@ -180,16 +222,22 @@ ThreadInfo[][] captureThreadStacks(ThreadMXBean threadBean, long[] threadIds) th
         return result;
     }
 
-    private void setThreadWaitBlockTimeMonitoringEnabled(ThreadMXBean threadBean, boolean enabled) {
+    private boolean isThreadWaitBlockTimeMonitoringEnabled(ThreadMXBean threadBean) {
         if (threadBean.isThreadContentionMonitoringSupported()) {
-            AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
-                threadBean.setThreadContentionMonitoringEnabled(enabled);
-                return null;
-            });
+            return threadBean.isThreadContentionMonitoringEnabled();
         }
+        return false;
     }
 
-    String innerDetect(ThreadMXBean threadBean, SunThreadInfo sunThreadInfo, long currentThreadId) throws Exception {
+    private double getTimeSharePercentage(long time) {
+        return (((double) time) / interval.nanos()) * 100;
+    }
+
+    String innerDetect(
+        ThreadMXBean threadBean,
+        SunThreadInfo sunThreadInfo,
+        long currentThreadId,
+        SleepFunction<Long, Void> threadSleep) throws Exception {
         if (threadBean.isThreadCpuTimeSupported() == false) {
             throw new ElasticsearchException("thread CPU time is not supported on this JDK");
         }
@@ -198,6 +246,12 @@ String innerDetect(ThreadMXBean threadBean, SunThreadInfo sunThreadInfo, long cu
             throw new ElasticsearchException("thread allocated memory is not supported on this JDK");
         }
 
+        // Enabling thread contention monitoring is required for capturing JVM thread wait/blocked times. If we weren't
+        // able to enable this functionality during bootstrap, we should not produce HotThreads reports.
+        if (isThreadWaitBlockTimeMonitoringEnabled(threadBean) == false) {
+            throw new ElasticsearchException("thread wait/blocked time accounting is not supported on this JDK");
+        }
+
         StringBuilder sb = new StringBuilder()
             .append("Hot threads at ")
             .append(DATE_TIME_FORMATTER.format(LocalDateTime.now(Clock.systemUTC())))
@@ -209,99 +263,115 @@ String innerDetect(ThreadMXBean threadBean, SunThreadInfo sunThreadInfo, long cu
             .append(ignoreIdleThreads)
             .append(":\n");
 
-        // Enabling thread contention monitoring is required for capturing JVM thread wait/blocked times
-        setThreadWaitBlockTimeMonitoringEnabled(threadBean, true);
+        // Capture before and after thread state with timings
+        Map<Long, ThreadTimeAccumulator> previousThreadInfos = getAllValidThreadInfos(threadBean, sunThreadInfo, currentThreadId);
+        threadSleep.apply(interval.millis());
+        Map<Long, ThreadTimeAccumulator> latestThreadInfos = getAllValidThreadInfos(threadBean, sunThreadInfo, currentThreadId);
 
-        try {
-            // Capture before and after thread state with timings
-            Map<Long, ThreadTimeAccumulator> previousThreadInfos = getAllValidThreadInfos(threadBean, sunThreadInfo, currentThreadId);
-            Thread.sleep(interval.millis());
-            Map<Long, ThreadTimeAccumulator> latestThreadInfos = getAllValidThreadInfos(threadBean, sunThreadInfo, currentThreadId);
+        latestThreadInfos.forEach((threadId, accumulator) -> accumulator.subtractPrevious(previousThreadInfos.get(threadId)));
 
-            latestThreadInfos.forEach((threadId, accumulator) -> accumulator.subtractPrevious(previousThreadInfos.get(threadId)));
+        // Sort by delta CPU time on thread.
+        List<ThreadTimeAccumulator> topThreads = new ArrayList<>(latestThreadInfos.values());
 
-            // Sort by delta CPU time on thread.
-            List<ThreadTimeAccumulator> topThreads = new ArrayList<>(latestThreadInfos.values());
-            final ToLongFunction<ThreadTimeAccumulator> getter = ThreadTimeAccumulator.valueGetterForReportType(type);
+        // Special comparator for CPU mode with TOTAL sort type only. Otherwise, we simply use the value.
+        if (type == ReportType.CPU && sortOrder == SortOrder.TOTAL)  {
+            CollectionUtil.introSort(topThreads,
+                Comparator.comparingLong(ThreadTimeAccumulator::getRunnableTime)
+                    .thenComparingLong(ThreadTimeAccumulator::getCpuTime).reversed());
+        } else {
+            CollectionUtil.introSort(topThreads,
+                Comparator.comparingLong(ThreadTimeAccumulator.valueGetterForReportType(type)).reversed());
+        }
 
-            CollectionUtil.introSort(topThreads, Comparator.comparingLong(getter).reversed());
-            topThreads = topThreads.subList(0, Math.min(busiestThreads, topThreads.size()));
-            long[] topThreadIds = topThreads.stream().mapToLong(t -> t.threadId).toArray();
+        topThreads = topThreads.subList(0, Math.min(busiestThreads, topThreads.size()));
+        long[] topThreadIds = topThreads.stream().mapToLong(t -> t.threadId).toArray();
 
-            // analyse N stack traces for the top threads
-            ThreadInfo[][] allInfos = captureThreadStacks(threadBean, topThreadIds);
+        // analyse N stack traces for the top threads
+        ThreadInfo[][] allInfos = captureThreadStacks(threadBean, topThreadIds);
 
-            for (int t = 0; t < topThreads.size(); t++) {
-                long timeOrBytes = getter.applyAsLong(topThreads.get(t));
-                String threadName = null;
-                for (ThreadInfo[] info : allInfos) {
-                    if (info != null && info[t] != null) {
-                        if (ignoreIdleThreads && isIdleThread(info[t])) {
-                            info[t] = null;
-                            continue;
-                        }
-                        threadName = info[t].getThreadName();
-                        break;
+        for (int t = 0; t < topThreads.size(); t++) {
+            String threadName = null;
+            for (ThreadInfo[] info : allInfos) {
+                if (info != null && info[t] != null) {
+                    if (ignoreIdleThreads && isIdleThread(info[t])) {
+                        info[t] = null;
+                        continue;
                     }
+                    threadName = info[t].getThreadName();
+                    break;
                 }
-                if (threadName == null) {
-                    continue; // thread is not alive yet or died before the first snapshot - ignore it!
-                }
+            }
+            if (threadName == null) {
+                continue; // thread is not alive yet or died before the first snapshot - ignore it!
+            }
+
+            ThreadTimeAccumulator topThread = topThreads.get(t);
 
-                if (type == ReportType.MEM) {
+            switch (type) {
+                case MEM:
                     sb.append(String.format(Locale.ROOT, "%n%s memory allocated by thread '%s'%n",
-                        new ByteSizeValue(timeOrBytes), threadName));
-                } else {
-                    double percent = (((double) timeOrBytes) / interval.nanos()) * 100;
+                        new ByteSizeValue(topThread.getAllocatedBytes()), threadName));
+                    break;
+                case CPU:
+                    double percentCpu = getTimeSharePercentage(topThread.getCpuTime());
+                    double percentOther = getTimeSharePercentage(topThread.getOtherTime());
+                    sb.append(String.format(Locale.ROOT,
+                        "%n%4.1f%% [cpu=%1.1f%%, other=%1.1f%%] (%s out of %s) %s usage by thread '%s'%n",
+                        percentOther + percentCpu, percentCpu, percentOther,
+                        TimeValue.timeValueNanos(topThread.getCpuTime() + topThread.getOtherTime()),
+                        interval, type.getTypeValue(), threadName));
+                    break;
+                default:
+                    long time = ThreadTimeAccumulator.valueGetterForReportType(type).applyAsLong(topThread);
+                    double percent = getTimeSharePercentage(time);
                     sb.append(String.format(Locale.ROOT, "%n%4.1f%% (%s out of %s) %s usage by thread '%s'%n",
-                        percent, TimeValue.timeValueNanos(timeOrBytes), interval, type.getTypeValue(), threadName));
+                        percent, TimeValue.timeValueNanos(time), interval, type.getTypeValue(), threadName));
+                    break;
+            }
+
+            // for each snapshot (2nd array index) find later snapshot for same thread with max number of
+            // identical StackTraceElements (starting from end of each)
+            boolean[] done = new boolean[threadElementsSnapshotCount];
+            for (int i = 0; i < threadElementsSnapshotCount; i++) {
+                if (done[i]) continue;
+                int maxSim = 1;
+                boolean[] similars = new boolean[threadElementsSnapshotCount];
+                for (int j = i + 1; j < threadElementsSnapshotCount; j++) {
+                    if (done[j]) continue;
+                    int similarity = similarity(allInfos[i][t], allInfos[j][t]);
+                    if (similarity > maxSim) {
+                        maxSim = similarity;
+                        similars = new boolean[threadElementsSnapshotCount];
+                    }
+                    if (similarity == maxSim) similars[j] = true;
                 }
-                // for each snapshot (2nd array index) find later snapshot for same thread with max number of
-                // identical StackTraceElements (starting from end of each)
-                boolean[] done = new boolean[threadElementsSnapshotCount];
-                for (int i = 0; i < threadElementsSnapshotCount; i++) {
-                    if (done[i]) continue;
-                    int maxSim = 1;
-                    boolean[] similars = new boolean[threadElementsSnapshotCount];
-                    for (int j = i + 1; j < threadElementsSnapshotCount; j++) {
-                        if (done[j]) continue;
-                        int similarity = similarity(allInfos[i][t], allInfos[j][t]);
-                        if (similarity > maxSim) {
-                            maxSim = similarity;
-                            similars = new boolean[threadElementsSnapshotCount];
-                        }
-                        if (similarity == maxSim) similars[j] = true;
+                // print out trace maxSim levels of i, and mark similar ones as done
+                int count = 1;
+                for (int j = i + 1; j < threadElementsSnapshotCount; j++) {
+                    if (similars[j]) {
+                        done[j] = true;
+                        count++;
                     }
-                    // print out trace maxSim levels of i, and mark similar ones as done
-                    int count = 1;
-                    for (int j = i + 1; j < threadElementsSnapshotCount; j++) {
-                        if (similars[j]) {
-                            done[j] = true;
-                            count++;
+                }
+                if (allInfos[i][t] != null) {
+                    final StackTraceElement[] show = allInfos[i][t].getStackTrace();
+                    if (count == 1) {
+                        sb.append(String.format(Locale.ROOT, "  unique snapshot%n"));
+                        for (StackTraceElement frame : show) {
+                            sb.append(String.format(Locale.ROOT, "    %s%n", frame));
                         }
-                    }
-                    if (allInfos[i][t] != null) {
-                        final StackTraceElement[] show = allInfos[i][t].getStackTrace();
-                        if (count == 1) {
-                            sb.append(String.format(Locale.ROOT, "  unique snapshot%n"));
-                            for (StackTraceElement frame : show) {
-                                sb.append(String.format(Locale.ROOT, "    %s%n", frame));
-                            }
-                        } else {
-                            sb.append(String.format(Locale.ROOT, "  %d/%d snapshots sharing following %d elements%n",
-                                count, threadElementsSnapshotCount, maxSim));
-                            for (int l = show.length - maxSim; l < show.length; l++) {
-                                sb.append(String.format(Locale.ROOT, "    %s%n", show[l]));
-                            }
+                    } else {
+                        sb.append(String.format(Locale.ROOT, "  %d/%d snapshots sharing following %d elements%n",
+                            count, threadElementsSnapshotCount, maxSim));
+                        for (int l = show.length - maxSim; l < show.length; l++) {
+                            sb.append(String.format(Locale.ROOT, "    %s%n", show[l]));
                         }
                     }
                 }
             }
-
-            return sb.toString();
-        } finally {
-            setThreadWaitBlockTimeMonitoringEnabled(threadBean, false);
         }
+
+        return sb.toString();
     }
 
     int similarity(ThreadInfo threadInfo, ThreadInfo threadInfo0) {
@@ -320,18 +390,24 @@ int similarity(ThreadInfo threadInfo, ThreadInfo threadInfo0) {
 
     static class ThreadTimeAccumulator {
         private final long threadId;
+        private final TimeValue interval;
 
         private long cpuTime;
         private long blockedTime;
         private long waitedTime;
         private long allocatedBytes;
 
-        ThreadTimeAccumulator(ThreadInfo info, long cpuTime, long allocatedBytes) {
-            this.blockedTime = info.getBlockedTime();
-            this.waitedTime = info.getWaitedTime();
+        ThreadTimeAccumulator(ThreadInfo info, TimeValue interval, long cpuTime, long allocatedBytes) {
+            this.blockedTime = millisecondsToNanos(info.getBlockedTime()); // Convert to nanos to standardize
+            this.waitedTime = millisecondsToNanos(info.getWaitedTime()); // Convert to nanos to standardize
             this.cpuTime = cpuTime;
             this.allocatedBytes = allocatedBytes;
             this.threadId = info.getThreadId();
+            this.interval = interval;
+        }
+
+        private long millisecondsToNanos(long millis) {
+            return millis * 1_000_000;
         }
 
         void subtractPrevious(ThreadTimeAccumulator previous) {
@@ -354,6 +430,25 @@ public long getCpuTime() {
             return Math.max(cpuTime, 0);
         }
 
+        public long getRunnableTime() {
+            // If the thread didn't have any CPU movement, we can't really tell if it's
+            // not running, or it has been asleep forever.
+            if (getCpuTime() == 0) {
+                return 0;
+            }
+            return Math.max(interval.nanos() - getWaitedTime() - getBlockedTime(), 0);
+        }
+
+        public long getOtherTime() {
+            // If the thread didn't have any CPU movement, we can't really tell if it's
+            // not running, or it has been asleep forever.
+            if (getCpuTime() == 0) {
+                return 0;
+            }
+
+            return Math.max(getRunnableTime() - getCpuTime(), 0);
+        }
+
         public long getBlockedTime() {
             return Math.max(blockedTime, 0);
         }
@@ -384,4 +479,23 @@ static ToLongFunction<ThreadTimeAccumulator> valueGetterForReportType(ReportType
             throw new IllegalArgumentException("expected thread type to be either 'cpu', 'wait', 'mem', or 'block', but was " + type);
         }
     }
+
+    @FunctionalInterface
+    public interface SleepFunction<T, R> {
+        R apply(T t) throws InterruptedException;
+    }
+
+    public static void initializeRuntimeMonitoring() {
+        // We'll let the JVM boot without this functionality, however certain APIs like HotThreads will not
+        // function and report an error.
+        if (ManagementFactory.getThreadMXBean().isThreadContentionMonitoringSupported() == false) {
+            LogManager.getLogger(HotThreads.class).info("Thread wait/blocked time accounting not supported.");
+        } else {
+            try {
+                ManagementFactory.getThreadMXBean().setThreadContentionMonitoringEnabled(true);
+            } catch (UnsupportedOperationException monitoringUnavailable) {
+                LogManager.getLogger(HotThreads.class).warn("Thread wait/blocked time accounting cannot be enabled.");
+            }
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index d3eeb8e970df1..faacf6592796f 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -46,6 +46,7 @@
 import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
+import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
 import org.elasticsearch.cluster.metadata.SystemIndexMetadataUpgradeService;
 import org.elasticsearch.cluster.metadata.TemplateUpgradeService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -171,6 +172,7 @@
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.upgrades.SystemIndexMigrationExecutor;
 import org.elasticsearch.usage.UsageService;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
@@ -321,7 +323,7 @@ protected Node(final Environment initialEnvironment,
             if (Environment.PATH_SHARED_DATA_SETTING.exists(tmpSettings)) {
                 // NOTE: this must be done with an explicit check here because the deprecation property on a path setting will
                 // cause ES to fail to start since logging is not yet initialized on first read of the setting
-                deprecationLogger.critical(
+                deprecationLogger.warn(
                     DeprecationCategory.SETTINGS,
                     "shared-data-path",
                     "setting [path.shared_data] is deprecated and will be removed in a future release"
@@ -330,13 +332,13 @@ protected Node(final Environment initialEnvironment,
 
             if (initialEnvironment.dataFiles().length > 1) {
                 // NOTE: we use initialEnvironment here, but assertEquivalent below ensures the data paths do not change
-                deprecationLogger.critical(DeprecationCategory.SETTINGS, "multiple-data-paths",
+                deprecationLogger.warn(DeprecationCategory.SETTINGS, "multiple-data-paths",
                     "Configuring multiple [path.data] paths is deprecated. Use RAID or other system level features for utilizing " +
                         "multiple disks. This feature will be removed in a future release.");
             }
             if (Environment.dataPathUsesList(tmpSettings)) {
                 // already checked for multiple values above, so if this is a list it is a single valued list
-                deprecationLogger.critical(DeprecationCategory.SETTINGS, "multiple-data-paths-list",
+                deprecationLogger.warn(DeprecationCategory.SETTINGS, "multiple-data-paths-list",
                     "Configuring [path.data] with a list is deprecated. Instead specify as a string value.");
             }
 
@@ -429,7 +431,8 @@ protected Node(final Environment initialEnvironment,
                 searchModule.getNamedWriteables().stream(),
                 pluginsService.filterPlugins(Plugin.class).stream()
                     .flatMap(p -> p.getNamedWriteables().stream()),
-                ClusterModule.getNamedWriteables().stream())
+                ClusterModule.getNamedWriteables().stream(),
+                SystemIndexMigrationExecutor.getNamedWriteables().stream())
                 .flatMap(Function.identity()).collect(Collectors.toList());
             final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables);
             NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(Stream.of(
@@ -566,6 +569,15 @@ protected Node(final Environment initialEnvironment,
                 new MetadataCreateDataStreamService(threadPool, clusterService, metadataCreateIndexService);
             final MetadataDataStreamsService metadataDataStreamsService = new MetadataDataStreamsService(clusterService, indicesService);
 
+            final MetadataUpdateSettingsService metadataUpdateSettingsService = new MetadataUpdateSettingsService(
+                clusterService,
+                clusterModule.getAllocationService(),
+                settingsModule.getIndexScopedSettings(),
+                indicesService,
+                shardLimitValidator,
+                threadPool
+            );
+
             Collection<Object> pluginComponents = pluginsService.filterPlugins(Plugin.class).stream()
                 .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService,
                     scriptService, xContentRegistry, environment, nodeEnvironment,
@@ -651,14 +663,31 @@ protected Node(final Environment initialEnvironment,
                 threadPool, scriptService, bigArrays, searchModule.getFetchPhase(),
                 responseCollectorService, circuitBreakerService, executorSelector);
 
-            final List<PersistentTasksExecutor<?>> tasksExecutors = pluginsService
-                .filterPlugins(PersistentTaskPlugin.class).stream()
-                .map(p -> p.getPersistentTasksExecutor(clusterService, threadPool, client, settingsModule,
-                    clusterModule.getIndexNameExpressionResolver()))
+            final SystemIndexMigrationExecutor systemIndexMigrationExecutor = new SystemIndexMigrationExecutor(
+                client,
+                clusterService,
+                systemIndices,
+                metadataUpdateSettingsService,
+                metadataCreateIndexService,
+                settingsModule.getIndexScopedSettings());
+            final List<PersistentTasksExecutor<?>> builtinTaskExecutors = Arrays.asList(systemIndexMigrationExecutor);
+            final List<PersistentTasksExecutor<?>> pluginTaskExectors = pluginsService.filterPlugins(PersistentTaskPlugin.class)
+                .stream()
+                .map(
+                    p -> p.getPersistentTasksExecutor(
+                        clusterService,
+                        threadPool,
+                        client,
+                        settingsModule,
+                        clusterModule.getIndexNameExpressionResolver()
+                    )
+                )
                 .flatMap(List::stream)
                 .collect(toList());
-
-            final PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(tasksExecutors);
+            final List<PersistentTasksExecutor<?>> allTasksExectors = Stream.of(pluginTaskExectors, builtinTaskExecutors)
+                .flatMap(List::stream)
+                .collect(toList());
+            final PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(allTasksExectors);
             final PersistentTasksClusterService persistentTasksClusterService =
                 new PersistentTasksClusterService(settings, registry, clusterService, threadPool);
             resourcesToClose.add(persistentTasksClusterService);
@@ -697,6 +726,7 @@ protected Node(final Environment initialEnvironment,
                     b.bind(MetadataCreateIndexService.class).toInstance(metadataCreateIndexService);
                     b.bind(MetadataCreateDataStreamService.class).toInstance(metadataCreateDataStreamService);
                     b.bind(MetadataDataStreamsService.class).toInstance(metadataDataStreamsService);
+                    b.bind(MetadataUpdateSettingsService.class).toInstance(metadataUpdateSettingsService);
                     b.bind(SearchService.class).toInstance(searchService);
                     b.bind(SearchTransportService.class).toInstance(searchTransportService);
                     b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController(searchService::aggReduceContextBuilder));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
index fb85bd481e556..8da1376905ea5 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
@@ -91,9 +91,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
         NodesHotThreadsRequest nodesHotThreadsRequest = new NodesHotThreadsRequest(nodesIds);
         nodesHotThreadsRequest.threads(request.paramAsInt("threads", nodesHotThreadsRequest.threads()));
-        nodesHotThreadsRequest.ignoreIdleThreads(request.paramAsBoolean("ignore_idle_threads", nodesHotThreadsRequest.ignoreIdleThreads()));
+        nodesHotThreadsRequest.ignoreIdleThreads(
+            request.paramAsBoolean("ignore_idle_threads", nodesHotThreadsRequest.ignoreIdleThreads()));
         nodesHotThreadsRequest.type(HotThreads.ReportType.of(request.param("type", nodesHotThreadsRequest.type().getTypeValue())));
-        nodesHotThreadsRequest.interval(TimeValue.parseTimeValue(request.param("interval"), nodesHotThreadsRequest.interval(), "interval"));
+        nodesHotThreadsRequest.sortOrder(
+            HotThreads.SortOrder.of(request.param("sort", nodesHotThreadsRequest.sortOrder().getOrderValue())));
+        nodesHotThreadsRequest.interval(
+            TimeValue.parseTimeValue(request.param("interval"), nodesHotThreadsRequest.interval(), "interval"));
         nodesHotThreadsRequest.snapshots(request.paramAsInt("snapshots", nodesHotThreadsRequest.snapshots()));
         nodesHotThreadsRequest.timeout(request.param("timeout"));
         return channel -> client.admin().cluster().nodesHotThreads(
diff --git a/server/src/main/java/org/elasticsearch/script/DocBasedScript.java b/server/src/main/java/org/elasticsearch/script/DocBasedScript.java
index 8ef8104652aed..6a05eb0b70ffb 100644
--- a/server/src/main/java/org/elasticsearch/script/DocBasedScript.java
+++ b/server/src/main/java/org/elasticsearch/script/DocBasedScript.java
@@ -23,14 +23,14 @@ public DocBasedScript(DocReader docReader) {
         this.docReader = docReader;
     }
 
-    public Field<?> field(String fieldName) {
+    public Field field(String fieldName) {
         if (docReader == null) {
-            return new EmptyField<>(fieldName);
+            return new EmptyField(fieldName);
         }
         return docReader.field(fieldName);
     }
 
-    public Stream<Field<?>> fields(String fieldGlob) {
+    public Stream<Field> fields(String fieldGlob) {
         if (docReader == null) {
             return Stream.empty();
         }
diff --git a/server/src/main/java/org/elasticsearch/script/DocReader.java b/server/src/main/java/org/elasticsearch/script/DocReader.java
index 19f6f157f542b..17ea73c576026 100644
--- a/server/src/main/java/org/elasticsearch/script/DocReader.java
+++ b/server/src/main/java/org/elasticsearch/script/DocReader.java
@@ -22,10 +22,10 @@
  */
 public interface DocReader {
     /** New-style field access */
-    Field<?> field(String fieldName);
+    Field field(String fieldName);
 
     /** New-style field iterator */
-    Stream<Field<?>> fields(String fieldGlob);
+    Stream<Field> fields(String fieldGlob);
 
     /** Set the underlying docId */
     void setDocument(int docID);
diff --git a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java
index b295736fdb8d4..0462292f67ddd 100644
--- a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java
+++ b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java
@@ -39,11 +39,11 @@ public DocValuesDocReader(SearchLookup searchLookup, LeafReaderContext leafConte
     }
 
     @Override
-    public Field<?> field(String fieldName) {
+    public Field field(String fieldName) {
         LeafDocLookup leafDocLookup = leafSearchLookup.doc();
 
         if (leafDocLookup.containsKey(fieldName) == false) {
-            return new EmptyField<>(fieldName);
+            return new EmptyField(fieldName);
         }
 
         return leafDocLookup.getScriptField(fieldName);
@@ -51,7 +51,7 @@ public Field<?> field(String fieldName) {
 
 
     @Override
-    public Stream<Field<?>> fields(String fieldGlob) {
+    public Stream<Field> fields(String fieldGlob) {
         throw new UnsupportedOperationException("not implemented");
     }
 
diff --git a/server/src/main/java/org/elasticsearch/script/field/DocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/DocValuesField.java
index 9a48c3aa8f871..a412f4dc2a08a 100644
--- a/server/src/main/java/org/elasticsearch/script/field/DocValuesField.java
+++ b/server/src/main/java/org/elasticsearch/script/field/DocValuesField.java
@@ -10,7 +10,7 @@
 
 import java.io.IOException;
 
-public interface DocValuesField<T> extends Field<T> {
+public interface DocValuesField extends Field {
 
     /** Set the current document ID. */
     void setNextDocId(int docId) throws IOException;
diff --git a/server/src/main/java/org/elasticsearch/script/field/EmptyField.java b/server/src/main/java/org/elasticsearch/script/field/EmptyField.java
index 968a82ebc9660..c2b7252af5bfd 100644
--- a/server/src/main/java/org/elasticsearch/script/field/EmptyField.java
+++ b/server/src/main/java/org/elasticsearch/script/field/EmptyField.java
@@ -8,13 +8,10 @@
 
 package org.elasticsearch.script.field;
 
-import java.util.Collections;
-import java.util.List;
-
 /**
  * Script field with no mapping, always returns {@code defaultValue}.
  */
-public class EmptyField<T> implements Field<T> {
+public class EmptyField implements Field {
 
     private final String name;
 
@@ -36,19 +33,4 @@ public boolean isEmpty() {
     public int size() {
         return 0;
     }
-
-    @Override
-    public List<T> getValues() {
-        return Collections.emptyList();
-    }
-
-    @Override
-    public T getValue(T defaultValue) {
-        return defaultValue;
-    }
-
-    @Override
-    public T getValue(int index, T defaultValue) {
-        return defaultValue;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/script/field/Field.java b/server/src/main/java/org/elasticsearch/script/field/Field.java
index 3d927afd3dd32..e13732fa46785 100644
--- a/server/src/main/java/org/elasticsearch/script/field/Field.java
+++ b/server/src/main/java/org/elasticsearch/script/field/Field.java
@@ -9,10 +9,8 @@
 
 package org.elasticsearch.script.field;
 
-import java.util.List;
-
 /** A field in a document accessible via scripting. */
-public interface Field<T> {
+public interface Field {
 
     /** Returns the name of this field. */
     String getName();
@@ -22,13 +20,4 @@ public interface Field<T> {
 
     /** Returns the number of values this field has. */
     int size();
-
-    /** Get all values of a multivalued field.  If {@code isEmpty()} this returns an empty list. */
-    List<T> getValues();
-
-    /** Get the first value of a field, if {@code isEmpty()} return defaultValue instead */
-    T getValue(T defaultValue);
-
-    /** Get the value of a field as the specified index. */
-    T getValue(int index, T defaultValue);
 }
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
index 9352e91341c9b..d2198b237548d 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
@@ -25,7 +25,7 @@
 
 public class LeafDocLookup implements Map<String, ScriptDocValues<?>> {
 
-    private final Map<String, DocValuesField<?>> localCacheScriptFieldData = new HashMap<>(4);
+    private final Map<String, DocValuesField> localCacheScriptFieldData = new HashMap<>(4);
     private final Map<String, ScriptDocValues<?>> localCacheFieldData = new HashMap<>(4);
     private final Function<String, MappedFieldType> fieldTypeLookup;
     private final Function<MappedFieldType, IndexFieldData<?>> fieldDataLookup;
@@ -45,8 +45,8 @@ public void setDocument(int docId) {
         this.docId = docId;
     }
 
-    public DocValuesField<?> getScriptField(String fieldName) {
-        DocValuesField<?> field = localCacheScriptFieldData.get(fieldName);
+    public DocValuesField getScriptField(String fieldName) {
+        DocValuesField field = localCacheScriptFieldData.get(fieldName);
 
         if (field == null) {
             final MappedFieldType fieldType = fieldTypeLookup.apply(fieldName);
@@ -57,9 +57,9 @@ public DocValuesField<?> getScriptField(String fieldName) {
 
             // Load the field data on behalf of the script. Otherwise, it would require
             // additional permissions to deal with pagedbytes/ramusagestimator/etc.
-            field = AccessController.doPrivileged(new PrivilegedAction<DocValuesField<?>>() {
+            field = AccessController.doPrivileged(new PrivilegedAction<DocValuesField>() {
                 @Override
-                public DocValuesField<?> run() {
+                public DocValuesField run() {
                     return fieldDataLookup.apply(fieldType).load(reader).getScriptField(fieldName);
                 }
             });
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java
index 6ccf879755316..d4b978fce4d7c 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java
@@ -98,7 +98,7 @@ public void handleException(TransportException exp) {
         }
     }
 
-    static class ProxyRequest<T extends TransportRequest> extends TransportRequest {
+    static class ProxyRequest<T extends TransportRequest> extends TransportRequest implements RawIndexingDataTransportRequest {
         final T wrapped;
         final DiscoveryNode targetNode;
 
@@ -120,6 +120,14 @@ public void writeTo(StreamOutput out) throws IOException {
             targetNode.writeTo(out);
             wrapped.writeTo(out);
         }
+
+        @Override
+        public boolean isRawIndexingData() {
+            if (wrapped instanceof RawIndexingDataTransportRequest) {
+                return ((RawIndexingDataTransportRequest) wrapped).isRawIndexingData();
+            }
+            return false;
+        }
     }
 
     private static class CancellableProxyRequest<T extends TransportRequest> extends ProxyRequest<T> {
diff --git a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java b/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java
new file mode 100644
index 0000000000000..dc89e88605d4b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+/**
+ * Holds the results of the most recent attempt to migrate system indices. Updated by {@link SystemIndexMigrator} as it finishes each
+ * feature, or fails.
+ */
+public class FeatureMigrationResults implements Metadata.Custom {
+    public static final String TYPE = "system_index_migration";
+    private static final Version MIGRATION_ADDED_VERSION = Version.V_8_0_0;
+
+    private static final ParseField RESULTS_FIELD = new ParseField("results");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<FeatureMigrationResults, Void> PARSER = new ConstructingObjectParser<>(TYPE, a -> {
+        final Map<String, SingleFeatureMigrationResult> statuses = ((List<Tuple<String, SingleFeatureMigrationResult>>) a[0]).stream()
+            .collect(Collectors.toMap(Tuple::v1, Tuple::v2));
+        return new FeatureMigrationResults(statuses);
+    });
+
+    static {
+        PARSER.declareNamedObjects(
+            ConstructingObjectParser.constructorArg(),
+            (p, c, n) -> new Tuple<>(n, SingleFeatureMigrationResult.fromXContent(p)),
+            v -> { throw new IllegalArgumentException("ordered " + RESULTS_FIELD.getPreferredName() + " are not supported"); },
+            RESULTS_FIELD
+        );
+    }
+
+    private final Map<String, SingleFeatureMigrationResult> featureStatuses;
+
+    public FeatureMigrationResults(Map<String, SingleFeatureMigrationResult> featureStatuses) {
+        this.featureStatuses = Objects.requireNonNullElse(featureStatuses, new HashMap<>());
+    }
+
+    public FeatureMigrationResults(StreamInput in) throws IOException {
+        this.featureStatuses = in.readMap(StreamInput::readString, SingleFeatureMigrationResult::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(
+            featureStatuses,
+            (StreamOutput outStream, String featureName) -> outStream.writeString(featureName),
+            (StreamOutput outStream, SingleFeatureMigrationResult featureStatus) -> featureStatus.writeTo(outStream)
+        );
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(RESULTS_FIELD.getPreferredName(), featureStatuses);
+        return builder;
+    }
+
+    public static FeatureMigrationResults fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    /**
+     * Gets a map of feature name to that feature's status. Only contains features which have either been migrated successfully or
+     * failed to migrate.
+     * @return An unmodifiable map of feature names to migration statuses.
+     */
+    public Map<String, SingleFeatureMigrationResult> getFeatureStatuses() {
+        return Collections.unmodifiableMap(featureStatuses);
+    }
+
+    /**
+     * Convenience method for updating the results of a migration run. Produces a new {@link FeatureMigrationResults} updated with the
+     * given status for the given feature name.
+     * @param featureName The feature name to update. If this feature name is already present, its status will be overwritten.
+     * @param status The status that should be associated with the given {@code featureName}.
+     * @return A new {@link FeatureMigrationResults} with the given status associated with the given feature name. Other entries in the
+     *         map are unchanged.
+     */
+    public FeatureMigrationResults withResult(String featureName, SingleFeatureMigrationResult status) {
+        Map<String, SingleFeatureMigrationResult> newMap = new HashMap<>(featureStatuses);
+        newMap.put(featureName, status);
+        return new FeatureMigrationResults(newMap);
+    }
+
+    @Override
+    public EnumSet<Metadata.XContentContext> context() {
+        return Metadata.ALL_CONTEXTS;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return MIGRATION_ADDED_VERSION;
+    }
+
+    @Override
+    public String toString() {
+        return "SystemIndexMigrationResult{" + "featureStatuses=" + featureStatuses + '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if ((o instanceof FeatureMigrationResults) == false) return false;
+        FeatureMigrationResults that = (FeatureMigrationResults) o;
+        return featureStatuses.equals(that.featureStatuses);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(featureStatuses);
+    }
+
+    @Override
+    public Diff<Metadata.Custom> diff(Metadata.Custom previousState) {
+        return new ResultsDiff((FeatureMigrationResults) previousState, this);
+    }
+
+    public static NamedDiff<Metadata.Custom> readDiffFrom(StreamInput in) throws IOException {
+        return new ResultsDiff(in);
+    }
+
+    public static class ResultsDiff implements NamedDiff<Metadata.Custom> {
+        private final Diff<Map<String, SingleFeatureMigrationResult>> resultsDiff;
+
+        public ResultsDiff(FeatureMigrationResults before, FeatureMigrationResults after) {
+            this.resultsDiff = DiffableUtils.diff(before.featureStatuses, after.featureStatuses, DiffableUtils.getStringKeySerializer());
+        }
+
+        public ResultsDiff(StreamInput in) throws IOException {
+            this.resultsDiff = DiffableUtils.readJdkMapDiff(
+                in,
+                DiffableUtils.getStringKeySerializer(),
+                SingleFeatureMigrationResult::new,
+                ResultsDiff::readDiffFrom
+            );
+        }
+
+        @Override
+        public Metadata.Custom apply(Metadata.Custom part) {
+            TreeMap<String, SingleFeatureMigrationResult> newResults = new TreeMap<>(
+                resultsDiff.apply(((FeatureMigrationResults) part).featureStatuses)
+            );
+            return new FeatureMigrationResults(newResults);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            resultsDiff.writeTo(out);
+        }
+
+        static Diff<SingleFeatureMigrationResult> readDiffFrom(StreamInput in) throws IOException {
+            return AbstractDiffable.readDiffFrom(SingleFeatureMigrationResult::new, in);
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/upgrades/MigrationResultsUpdateTask.java b/server/src/main/java/org/elasticsearch/upgrades/MigrationResultsUpdateTask.java
new file mode 100644
index 0000000000000..8fb3278e2b7b2
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/upgrades/MigrationResultsUpdateTask.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
+
+import java.util.HashMap;
+
+/**
+ * Handles updating the {@link FeatureMigrationResults} in the cluster state.
+ */
+public class MigrationResultsUpdateTask extends ClusterStateUpdateTask {
+    private static final Logger logger = LogManager.getLogger(MigrationResultsUpdateTask.class);
+
+    private final String featureName;
+    private final SingleFeatureMigrationResult status;
+    private final ActionListener<ClusterState> listener;
+
+    private MigrationResultsUpdateTask(String featureName, SingleFeatureMigrationResult status, ActionListener<ClusterState> listener) {
+        this.featureName = featureName;
+        this.status = status;
+        this.listener = listener;
+    }
+
+    /**
+     * Creates a task that will update the status of a feature migration.
+     * @param featureName The name of the feature whose status should be updated.
+     * @param status The status to be associated with the given feature.
+     * @param listener A listener that will be called upon successfully updating the cluster state.
+     */
+    public static MigrationResultsUpdateTask upsert(
+        String featureName,
+        SingleFeatureMigrationResult status,
+        ActionListener<ClusterState> listener
+    ) {
+        return new MigrationResultsUpdateTask(featureName, status, listener);
+    }
+
+    /**
+     * Submit the update task so that it will actually be executed.
+     * @param clusterService The cluster service to which this task should be submitted.
+     */
+    public void submit(ClusterService clusterService) {
+        String source = new ParameterizedMessage("record [{}] migration [{}]", featureName, status.succeeded() ? "success" : "failure")
+            .getFormattedMessage();
+        clusterService.submitStateUpdateTask(source, this);
+    }
+
+    @Override
+    public ClusterState execute(ClusterState currentState) throws Exception {
+        FeatureMigrationResults currentResults = currentState.metadata().custom(FeatureMigrationResults.TYPE);
+        if (currentResults == null) {
+            currentResults = new FeatureMigrationResults(new HashMap<>());
+        }
+        FeatureMigrationResults newResults = currentResults.withResult(featureName, status);
+        final Metadata newMetadata = Metadata.builder(currentState.metadata())
+            .putCustom(FeatureMigrationResults.TYPE, newResults)
+            .build();
+        final ClusterState newState = ClusterState.builder(currentState).metadata(newMetadata).build();
+        return newState;
+    }
+
+    @Override
+    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+        listener.onResponse(newState);
+    }
+
+    @Override
+    public void onFailure(String source, Exception clusterStateUpdateException) {
+        if (status.succeeded()) {
+            logger.warn(
+                new ParameterizedMessage("failed to update cluster state after successful migration of feature [{}]", featureName),
+                clusterStateUpdateException
+            );
+        } else {
+            logger.error(
+                new ParameterizedMessage(
+                    "failed to update cluster state after failed migration of feature [{}] on index [{}]",
+                    featureName,
+                    status.getFailedIndexName()
+                ).getFormattedMessage(),
+                clusterStateUpdateException
+            );
+        }
+        listener.onFailure(clusterStateUpdateException);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/upgrades/SingleFeatureMigrationResult.java b/server/src/main/java/org/elasticsearch/upgrades/SingleFeatureMigrationResult.java
new file mode 100644
index 0000000000000..0c03e1426e1fe
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/upgrades/SingleFeatureMigrationResult.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Holds the results of migrating a single feature. See also {@link FeatureMigrationResults}.
+ */
+public class SingleFeatureMigrationResult extends AbstractDiffable<SingleFeatureMigrationResult> implements Writeable, ToXContent {
+    private static final String NAME = "feature_migration_status";
+    private static final ParseField SUCCESS_FIELD = new ParseField("successful");
+    private static final ParseField FAILED_INDEX_NAME_FIELD = new ParseField("failed_index");
+    private static final ParseField EXCEPTION_FIELD = new ParseField("exception");
+
+    private final boolean successful;
+    @Nullable
+    private final String failedIndexName;
+    @Nullable
+    private final Exception exception;
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<SingleFeatureMigrationResult, Void> PARSER = new ConstructingObjectParser<>(
+        NAME,
+        a -> new SingleFeatureMigrationResult((boolean) a[0], (String) a[1], (Exception) a[2])
+    );
+
+    static {
+        PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), SUCCESS_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), FAILED_INDEX_NAME_FIELD);
+        PARSER.declareObject(
+            ConstructingObjectParser.optionalConstructorArg(),
+            (p, c) -> ElasticsearchException.fromXContent(p),
+            EXCEPTION_FIELD
+        );
+    }
+
+    private SingleFeatureMigrationResult(boolean successful, String failedIndexName, Exception exception) {
+        this.successful = successful;
+        if (successful == false) {
+            Objects.requireNonNull(failedIndexName, "failed index name must be present for failed feature migration statuses");
+            Objects.requireNonNull(exception, "exception details must be present for failed feature migration statuses");
+        }
+        this.failedIndexName = failedIndexName;
+        this.exception = exception;
+    }
+
+    SingleFeatureMigrationResult(StreamInput in) throws IOException {
+        this.successful = in.readBoolean();
+        if (this.successful == false) {
+            this.failedIndexName = in.readString();
+            this.exception = in.readException();
+        } else {
+            this.failedIndexName = null;
+            this.exception = null;
+        }
+    }
+
+    public static SingleFeatureMigrationResult fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    /**
+     * Creates a record indicating that migration succeeded.
+     */
+    public static SingleFeatureMigrationResult success() {
+        return new SingleFeatureMigrationResult(true, null, null);
+    }
+
+    /**
+     * Creates a record indicating that migration failed.
+     * @param failedIndexName The name of the specific index whose migration failed.
+     * @param exception The exception which caused the migration failure.
+     */
+    public static SingleFeatureMigrationResult failure(String failedIndexName, Exception exception) {
+        return new SingleFeatureMigrationResult(false, failedIndexName, exception);
+    }
+
+    /**
+     * Returns {@code true} if the migration of this feature's data succeeded, or {@code false} otherwise.
+     */
+    public boolean succeeded() {
+        return successful;
+    }
+
+    /**
+     * Gets the name of the specific index where the migration failure occurred, if the migration failed.
+     */
+    @Nullable
+    public String getFailedIndexName() {
+        return failedIndexName;
+    }
+
+    /**
+     * Gets the exception that cause the migration failure, if the migration failed.
+     */
+    @Nullable
+    public Exception getException() {
+        return exception;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeBoolean(this.successful);
+        if (this.successful == false) {
+            out.writeString(this.failedIndexName);
+            out.writeException(this.exception);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.field(SUCCESS_FIELD.getPreferredName(), successful);
+            if (successful == false) {
+                builder.field(FAILED_INDEX_NAME_FIELD.getPreferredName(), failedIndexName);
+                builder.startObject(EXCEPTION_FIELD.getPreferredName());
+                {
+                    ElasticsearchException.generateThrowableXContent(builder, params, exception);
+                }
+                builder.endObject();
+            }
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean isFragment() {
+        return false;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if ((o instanceof SingleFeatureMigrationResult) == false) return false;
+        SingleFeatureMigrationResult that = (SingleFeatureMigrationResult) o;
+        // Exception is intentionally not checked here
+        return successful == that.successful && Objects.equals(failedIndexName, that.failedIndexName);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(successful, failedIndexName);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java
new file mode 100644
index 0000000000000..bf58eb029ffc0
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
+import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.persistent.AllocatedPersistentTask;
+import org.elasticsearch.persistent.PersistentTaskParams;
+import org.elasticsearch.persistent.PersistentTaskState;
+import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
+import org.elasticsearch.persistent.PersistentTasksExecutor;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME;
+
+/**
+ * Starts the process of migrating system indices. See {@link SystemIndexMigrator} for the actual migration logic.
+ */
+public class SystemIndexMigrationExecutor extends PersistentTasksExecutor<SystemIndexMigrationTaskParams> {
+    private final Client client; // NOTE: *NOT* an OriginSettingClient. We have to do that later.
+    private final ClusterService clusterService;
+    private final SystemIndices systemIndices;
+    private final MetadataUpdateSettingsService metadataUpdateSettingsService;
+    private final MetadataCreateIndexService metadataCreateIndexService;
+    private final IndexScopedSettings indexScopedSettings;
+
+    public SystemIndexMigrationExecutor(
+        Client client,
+        ClusterService clusterService,
+        SystemIndices systemIndices,
+        MetadataUpdateSettingsService metadataUpdateSettingsService,
+        MetadataCreateIndexService metadataCreateIndexService,
+        IndexScopedSettings indexScopedSettings) {
+        super(SYSTEM_INDEX_UPGRADE_TASK_NAME, ThreadPool.Names.GENERIC);
+        this.client = client;
+        this.clusterService = clusterService;
+        this.systemIndices = systemIndices;
+        this.metadataUpdateSettingsService = metadataUpdateSettingsService;
+        this.metadataCreateIndexService = metadataCreateIndexService;
+        this.indexScopedSettings = indexScopedSettings;
+    }
+
+    @Override
+    protected void nodeOperation(AllocatedPersistentTask task, SystemIndexMigrationTaskParams params, PersistentTaskState state) {
+        SystemIndexMigrator upgrader = (SystemIndexMigrator) task;
+        SystemIndexMigrationTaskState upgraderState = (SystemIndexMigrationTaskState) state;
+        upgrader.run(upgraderState);
+    }
+
+    @Override
+    protected AllocatedPersistentTask createTask(
+        long id,
+        String type,
+        String action,
+        TaskId parentTaskId,
+        PersistentTasksCustomMetadata.PersistentTask<SystemIndexMigrationTaskParams> taskInProgress,
+        Map<String, String> headers
+    ) {
+        return new SystemIndexMigrator(
+            client,
+            id,
+            type,
+            action,
+            parentTaskId,
+            taskInProgress.getParams(),
+            headers,
+            clusterService,
+            systemIndices,
+            metadataUpdateSettingsService,
+            metadataCreateIndexService,
+            indexScopedSettings);
+    }
+
+    @Override
+    public PersistentTasksCustomMetadata.Assignment getAssignment(
+        SystemIndexMigrationTaskParams params,
+        Collection<DiscoveryNode> candidateNodes,
+        ClusterState clusterState
+    ) {
+        // This should select from master-eligible nodes because we already require all master-eligible nodes to have all plugins installed.
+        // However, due to a misunderstanding, this code as-written needs to run on the master node in particular. This is not a fundamental
+        // problem, but more that you can't submit cluster state update tasks from non-master nodes. If we translate the process of updating
+        // the cluster state to a Transport action, we can revert this to selecting any master-eligible node.
+        DiscoveryNode discoveryNode = clusterState.nodes().getMasterNode();
+        if (discoveryNode == null) {
+            return NO_NODE_FOUND;
+        } else {
+            return new PersistentTasksCustomMetadata.Assignment(discoveryNode.getId(), "");
+        }
+    }
+
+    public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        return List.of(new NamedWriteableRegistry.Entry(
+                PersistentTaskState.class,
+                SYSTEM_INDEX_UPGRADE_TASK_NAME,
+                SystemIndexMigrationTaskState::new
+            ),
+            new NamedWriteableRegistry.Entry(
+                PersistentTaskParams.class,
+                SYSTEM_INDEX_UPGRADE_TASK_NAME,
+                SystemIndexMigrationTaskParams::new
+            ));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java
new file mode 100644
index 0000000000000..0892fa0d0bd01
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java
@@ -0,0 +1,284 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.OriginSettingClient;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.plugins.SystemIndexPlugin;
+
+import java.util.Comparator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.cluster.metadata.IndexMetadata.State.CLOSE;
+
+/**
+ * Holds the data required to migrate a single system index, including metadata from the current index. If necessary, computes the settings
+ * and mappings for the "next" index based off of the current one.
+ */
+class SystemIndexMigrationInfo implements Comparable<SystemIndexMigrationInfo> {
+    private static final Logger logger = LogManager.getLogger(SystemIndexMigrationInfo.class);
+
+    private final IndexMetadata currentIndex;
+    private final String featureName;
+    private final Settings settings;
+    private final String mapping;
+    private final String origin;
+    private final SystemIndices.Feature owningFeature;
+
+    private static final Comparator<SystemIndexMigrationInfo> SAME_CLASS_COMPARATOR = Comparator.comparing(
+        SystemIndexMigrationInfo::getFeatureName
+    ).thenComparing(SystemIndexMigrationInfo::getCurrentIndexName);
+
+    private SystemIndexMigrationInfo(
+        IndexMetadata currentIndex,
+        String featureName,
+        Settings settings,
+        String mapping,
+        String origin,
+        SystemIndices.Feature owningFeature
+    ) {
+        this.currentIndex = currentIndex;
+        this.featureName = featureName;
+        this.settings = settings;
+        this.mapping = mapping;
+        this.origin = origin;
+        this.owningFeature = owningFeature;
+    }
+
+    /**
+     * Gets the name of the index to be migrated.
+     */
+    String getCurrentIndexName() {
+        return currentIndex.getIndex().getName();
+    }
+
+    /**
+     * Indicates if the index to be migrated is closed.
+     */
+    boolean isCurrentIndexClosed() {
+        return CLOSE.equals(currentIndex.getState());
+    }
+
+    /**
+     * Gets the name to be used for the post-migration index.
+     */
+    String getNextIndexName() {
+        return currentIndex.getIndex().getName() + SystemIndices.UPGRADED_INDEX_SUFFIX;
+    }
+
+    /**
+     * Gets the name of the feature which owns the index to be migrated.
+     */
+    String getFeatureName() {
+        return featureName;
+    }
+
+    /**
+     * Gets the mappings to be used for the post-migration index.
+     */
+    String getMappings() {
+        return mapping;
+    }
+
+    /**
+     * Gets the settings to be used for the post-migration index.
+     */
+    Settings getSettings() {
+        return settings;
+    }
+
+    /**
+     * Gets the origin that should be used when interacting with this index.
+     */
+    String getOrigin() {
+        return origin;
+    }
+
+    /**
+     * Invokes the pre-migration hook for the feature that owns this index.
+     * See {@link SystemIndexPlugin#prepareForIndicesMigration(ClusterService, Client, ActionListener)}.
+     * @param clusterService For retrieving the state.
+     * @param client For performing any update operations necessary to prepare for the upgrade.
+     * @param listener Call {@link ActionListener#onResponse(Object)} when preparation for migration is complete.
+     */
+    void prepareForIndicesMigration(
+        ClusterService clusterService,
+        Client client,
+        ActionListener<Map<String, Object>> listener) {
+        owningFeature.getPreMigrationFunction().prepareForIndicesMigration(clusterService, client, listener);
+    }
+
+    /**
+     * Invokes the post-migration hooks for the feature that owns this index.
+     * See {@link SystemIndexPlugin#indicesMigrationComplete(Map, ClusterService, Client, ActionListener)}.
+     * @param metadata The metadata that was passed into the listener by the pre-migration hook.
+     * @param clusterService For retrieving the state.
+     * @param client For performing any update operations necessary to prepare for the upgrade.
+     * @param listener Call {@link ActionListener#onResponse(Object)} when the hook is finished.
+     */
+    void indicesMigrationComplete(
+        Map<String, Object> metadata,
+        ClusterService clusterService,
+        Client client,
+        ActionListener<Boolean> listener
+    ) {
+        owningFeature.getPostMigrationFunction().indicesMigrationComplete(metadata, clusterService, client, listener);
+    }
+
+    /**
+     * Creates a client that's been configured to be able to properly access the system index to be migrated.
+     * @param baseClient The base client to wrap.
+     * @return An {@link OriginSettingClient} which uses the origin provided by {@link SystemIndexMigrationInfo#getOrigin()}.
+     */
+    Client createClient(Client baseClient) {
+        return new OriginSettingClient(baseClient, this.getOrigin());
+    }
+
+    @Override
+    public int compareTo(SystemIndexMigrationInfo o) {
+        return SAME_CLASS_COMPARATOR.compare(this, o);
+    }
+
+    @Override
+    public String toString() {
+        return "IndexUpgradeInfo["
+            + "currentIndex='"
+            + currentIndex.getIndex().getName()
+            + "\'"
+            + ", featureName='"
+            + featureName
+            + '\''
+            + ", settings="
+            + settings
+            + ", mapping='"
+            + mapping
+            + '\''
+            + ", origin='"
+            + origin
+            + '\'';
+    }
+
+    static SystemIndexMigrationInfo build(
+        IndexMetadata currentIndex,
+        SystemIndexDescriptor descriptor,
+        SystemIndices.Feature feature,
+        IndexScopedSettings indexScopedSettings
+    ) {
+        Settings.Builder settingsBuilder = Settings.builder();
+        if (descriptor.getSettings() != null) {
+            settingsBuilder.put(descriptor.getSettings());
+            settingsBuilder.remove("index.version.created"); // Simplifies testing, should never impact real uses.
+        }
+        Settings settings = settingsBuilder.build();
+
+        String mapping = descriptor.getMappings();
+        if (descriptor.isAutomaticallyManaged() == false) {
+            // Get Settings from old index
+            settings = copySettingsForNewIndex(currentIndex.getSettings(), indexScopedSettings);
+
+            // Copy mapping from the old index
+            mapping = currentIndex.mapping().source().string();
+        }
+        return new SystemIndexMigrationInfo(
+            currentIndex,
+            feature.getName(),
+            settings,
+            mapping,
+            descriptor.getOrigin(),
+            feature);
+    }
+
+    private static Settings copySettingsForNewIndex(Settings currentIndexSettings, IndexScopedSettings indexScopedSettings) {
+        Settings.Builder newIndexSettings = Settings.builder();
+        currentIndexSettings.keySet()
+            .stream()
+            .filter(settingKey -> indexScopedSettings.isPrivateSetting(settingKey) == false)
+            .map(indexScopedSettings::get)
+            .filter(Objects::nonNull)
+            .filter(setting -> setting.getProperties().contains(Setting.Property.NotCopyableOnResize) == false)
+            .filter(setting -> setting.getProperties().contains(Setting.Property.PrivateIndex) == false)
+            .forEach(setting -> { newIndexSettings.put(setting.getKey(), currentIndexSettings.get(setting.getKey())); });
+        return newIndexSettings.build();
+    }
+
+    /**
+     * Convenience factory method holding the logic for creating instances from a Feature object.
+     * @param feature The feature that
+     * @param metadata The current metadata, as index migration depends on the current state of the clsuter.
+     * @param indexScopedSettings This is necessary to make adjustments to the indices settings for unmanaged indices.
+     * @return A {@link Stream} of {@link SystemIndexMigrationInfo}s that represent all the indices the given feature currently owns.
+     */
+    static Stream<SystemIndexMigrationInfo> fromFeature(
+        SystemIndices.Feature feature,
+        Metadata metadata,
+        IndexScopedSettings indexScopedSettings
+    ) {
+        return feature.getIndexDescriptors()
+            .stream()
+            .flatMap(descriptor -> descriptor.getMatchingIndices(metadata).stream().map(metadata::index).filter(imd -> {
+                assert imd != null : "got null IndexMetadata for index in system index descriptor [" + descriptor.getIndexPattern() + "]";
+                return Objects.nonNull(imd);
+            }).map(imd -> SystemIndexMigrationInfo.build(imd, descriptor, feature, indexScopedSettings)));
+    }
+
+    static SystemIndexMigrationInfo fromTaskState(
+        SystemIndexMigrationTaskState taskState,
+        SystemIndices systemIndices,
+        Metadata metadata,
+        IndexScopedSettings indexScopedSettings
+    ) {
+        SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(taskState.getCurrentIndex());
+        SystemIndices.Feature feature = systemIndices.getFeatures().get(taskState.getCurrentFeature());
+        IndexMetadata imd = metadata.index(taskState.getCurrentIndex());
+
+        // It's possible for one or both of these to happen if the executing node fails during execution and:
+        // 1. The task gets assigned to a node with a different set of plugins installed.
+        // 2. The index in question is somehow deleted before we got to it.
+        // The first case shouldn't happen, master nodes must have all `SystemIndexPlugins` installed.
+        // In the second case, we should just start over.
+        if (descriptor == null) {
+            String errorMsg = new ParameterizedMessage(
+                "couldn't find system index descriptor for index [{}] from feature [{}], which likely means this node is missing a plugin",
+                taskState.getCurrentIndex(),
+                taskState.getCurrentFeature()
+            ).toString();
+            logger.warn(errorMsg);
+            assert false : errorMsg;
+            throw new IllegalStateException(errorMsg);
+        }
+
+        if (imd == null) {
+            String errorMsg = new ParameterizedMessage(
+                "couldn't find index [{}] from feature [{}] with descriptor pattern [{}]",
+                taskState.getCurrentIndex(),
+                taskState.getCurrentFeature(),
+                descriptor.getIndexPattern()
+            ).toString();
+            logger.warn(errorMsg);
+            assert false : errorMsg;
+            throw new IllegalStateException(errorMsg);
+        }
+
+        return build(imd, descriptor, feature, indexScopedSettings);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java
new file mode 100644
index 0000000000000..3f63b29c64a79
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.persistent.PersistentTaskParams;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * The params used to initialize {@link SystemIndexMigrator} when it's initially kicked off.
+ *
+ * Currently doesn't do anything. In the future, a name of a feature could be used to indicate that only a specific feature should be
+ * migrated.
+ */
+public class SystemIndexMigrationTaskParams implements PersistentTaskParams {
+
+    public static final String SYSTEM_INDEX_UPGRADE_TASK_NAME = "upgrade-system-indices";
+    public static final ObjectParser<SystemIndexMigrationTaskParams, Void> PARSER = new ObjectParser<>(
+        SYSTEM_INDEX_UPGRADE_TASK_NAME,
+        true,
+        SystemIndexMigrationTaskParams::new
+    );
+    static {
+        // PARSER.declareString etc...
+    }
+
+    public SystemIndexMigrationTaskParams() {
+        // This space intentionally left blank
+    }
+
+    public SystemIndexMigrationTaskParams(StreamInput in) {
+        // This space intentionally left blank
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        // This space intentionally left blank
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return SYSTEM_INDEX_UPGRADE_TASK_NAME;
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return Version.V_8_0_0;
+    }
+
+    @Override public boolean equals(Object obj) {
+        return obj instanceof SystemIndexMigrationTaskParams;
+    }
+
+    @Override public int hashCode() {
+        return 0;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskState.java b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskState.java
new file mode 100644
index 0000000000000..06a5acedc61cc
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskState.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.persistent.PersistentTaskState;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+
+/**
+ * Contains the current state of system index migration progress. Used to resume runs if there's a node failure during migration.
+ */
+public class SystemIndexMigrationTaskState implements PersistentTaskState {
+    private static final ParseField CURRENT_INDEX_FIELD = new ParseField("current_index");
+    private static final ParseField CURRENT_FEATURE_FIELD = new ParseField("current_feature");
+    private static final ParseField FEATURE_METADATA_MAP_FIELD = new ParseField("feature_metadata");
+
+    @SuppressWarnings(value = "unchecked")
+    private static final ConstructingObjectParser<SystemIndexMigrationTaskState, Void> PARSER = new ConstructingObjectParser<>(
+        SYSTEM_INDEX_UPGRADE_TASK_NAME,
+        true,
+        args -> new SystemIndexMigrationTaskState((String) args[0], (String) args[1], (Map<String, Object>) args[3])
+    );
+
+    static {
+        PARSER.declareString(constructorArg(), CURRENT_INDEX_FIELD);
+        PARSER.declareString(constructorArg(), CURRENT_FEATURE_FIELD);
+        PARSER.declareObject(constructorArg(), (p, c) -> p.map(), FEATURE_METADATA_MAP_FIELD);
+    }
+
+    private final String currentIndex;
+    private final String currentFeature;
+    private final Map<String, Object> featureCallbackMetadata;
+
+    public SystemIndexMigrationTaskState(String currentIndex, String currentFeature, Map<String, Object> featureCallbackMetadata) {
+        this.currentIndex = Objects.requireNonNull(currentIndex);
+        this.currentFeature = Objects.requireNonNull(currentFeature);
+        this.featureCallbackMetadata = Objects.requireNonNullElse(featureCallbackMetadata, new HashMap<>());
+    }
+
+    public SystemIndexMigrationTaskState(StreamInput in) throws IOException {
+        this.currentIndex = in.readString();
+        this.currentFeature = in.readString();
+        this.featureCallbackMetadata = in.readMap();
+    }
+
+    /**
+     * Gets the name of the index that's currently being migrated.
+     */
+    public String getCurrentIndex() {
+        return currentIndex;
+    }
+
+    /**
+     * Gets the name of the feature which owns the index that's currently being migrated.
+     */
+    public String getCurrentFeature() {
+        return currentFeature;
+    }
+
+    /**
+     * Retrieves metadata stored by the pre-upgrade hook, intended for consumption by the post-migration hook.
+     * See {@link org.elasticsearch.plugins.SystemIndexPlugin#prepareForIndicesMigration(ClusterService, Client, ActionListener)} and
+     * {@link org.elasticsearch.plugins.SystemIndexPlugin#indicesMigrationComplete(Map, ClusterService, Client, ActionListener)} for
+     * details on the pre- and post-migration hooks.
+     */
+    public Map<String, Object> getFeatureCallbackMetadata() {
+        return featureCallbackMetadata;
+    }
+
+    public static SystemIndexMigrationTaskState fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.field(CURRENT_INDEX_FIELD.getPreferredName(), currentIndex);
+            builder.field(CURRENT_FEATURE_FIELD.getPreferredName(), currentFeature);
+            builder.field(FEATURE_METADATA_MAP_FIELD.getPreferredName(), featureCallbackMetadata);
+        }
+        builder.endObject();
+        return null;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return SYSTEM_INDEX_UPGRADE_TASK_NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(currentIndex);
+        out.writeString(currentFeature);
+        out.writeMap(featureCallbackMetadata);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if ((o instanceof SystemIndexMigrationTaskState) == false) return false;
+        SystemIndexMigrationTaskState that = (SystemIndexMigrationTaskState) o;
+        return currentIndex.equals(that.currentIndex)
+            && currentFeature.equals(that.currentFeature)
+            && featureCallbackMetadata.equals(that.featureCallbackMetadata);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(currentIndex, currentFeature, featureCallbackMetadata);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java
new file mode 100644
index 0000000000000..1164b0a0ba8ac
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java
@@ -0,0 +1,610 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
+import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsClusterStateUpdateRequest;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.ParentTaskAssigningClient;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
+import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.index.reindex.ReindexAction;
+import org.elasticsearch.index.reindex.ReindexRequest;
+import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.persistent.AllocatedPersistentTask;
+import org.elasticsearch.tasks.TaskId;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.State.CLOSE;
+
+/**
+ * This is where the logic to actually perform the migration lives - {@link SystemIndexMigrator#run(SystemIndexMigrationTaskState)} will
+ * be invoked when the migration process is started, plus any time the node running the migration drops from the cluster/crashes/etc.
+ *
+ * See {@link SystemIndexMigrationTaskState} for the data that's saved for node failover.
+ */
+public class SystemIndexMigrator extends AllocatedPersistentTask {
+    private static final Logger logger = LogManager.getLogger(SystemIndexMigrator.class);
+
+    private static final Version READY_FOR_MIGRATION_VERSION = Version.V_7_16_0;
+
+    // Fixed properties & services
+    private final ParentTaskAssigningClient baseClient;
+    private final ClusterService clusterService;
+    private final SystemIndices systemIndices;
+    private final MetadataUpdateSettingsService metadataUpdateSettingsService;
+    private final MetadataCreateIndexService metadataCreateIndexService;
+    private final IndexScopedSettings indexScopedSettings;
+
+    // In-memory state
+    // NOTE: This queue is not a thread-safe class. Use `synchronized (migrationQueue)` whenever you access this. I chose this rather than
+    // a synchronized/concurrent collection or an AtomicReference because we often need to do compound operations, which are much simpler
+    // with `synchronized` blocks than when only the collection accesses are protected.
+    private final Queue<SystemIndexMigrationInfo> migrationQueue = new LinkedList<>();
+    private final AtomicReference<Map<String, Object>> currentFeatureCallbackMetadata = new AtomicReference<>();
+
+    public SystemIndexMigrator(
+        Client client,
+        long id,
+        String type,
+        String action,
+        TaskId parentTask,
+        SystemIndexMigrationTaskParams params,
+        Map<String, String> headers,
+        ClusterService clusterService,
+        SystemIndices systemIndices,
+        MetadataUpdateSettingsService metadataUpdateSettingsService,
+        MetadataCreateIndexService metadataCreateIndexService,
+        IndexScopedSettings indexScopedSettings
+    ) {
+        super(id, type, action, "system-index-migrator", parentTask, headers);
+        this.baseClient = new ParentTaskAssigningClient(client, parentTask);
+        this.clusterService = clusterService;
+        this.systemIndices = systemIndices;
+        this.metadataUpdateSettingsService = metadataUpdateSettingsService;
+        this.metadataCreateIndexService = metadataCreateIndexService;
+        this.indexScopedSettings = indexScopedSettings;
+    }
+
+    public void run(SystemIndexMigrationTaskState taskState) {
+        ClusterState clusterState = clusterService.state();
+
+        final String stateIndexName;
+        final String stateFeatureName;
+
+        if (taskState != null) {
+            currentFeatureCallbackMetadata.set(taskState.getFeatureCallbackMetadata());
+            stateIndexName = taskState.getCurrentIndex();
+            stateFeatureName = taskState.getCurrentFeature();
+
+            SystemIndices.Feature feature = systemIndices.getFeatures().get(stateFeatureName);
+            if (feature == null) {
+                markAsFailed(
+                    new IllegalStateException(
+                        "cannot migrate feature [" + stateFeatureName + "] because that feature is not installed on this node"
+                    )
+                );
+                return;
+            }
+
+            if (stateIndexName != null && clusterState.metadata().hasIndex(stateIndexName) == false) {
+                markAsFailed(new IndexNotFoundException(stateIndexName, "cannot migrate because that index does not exist"));
+                return;
+            }
+        } else {
+            stateIndexName = null;
+            stateFeatureName = null;
+        }
+
+        synchronized (migrationQueue) {
+            if (migrationQueue.isEmpty() == false && taskState == null) {
+                // A null task state means this is a new execution, not resumed from a failure
+                markAsFailed(new IllegalStateException("migration is already in progress, cannot start new migration"));
+                return;
+            }
+
+            systemIndices.getFeatures()
+                .values()
+                .stream()
+                .flatMap(feature -> SystemIndexMigrationInfo.fromFeature(feature, clusterState.metadata(), indexScopedSettings))
+                .filter(migrationInfo -> needsToBeMigrated(clusterState.metadata().index(migrationInfo.getCurrentIndexName())))
+                .sorted() // Stable order between nodes
+                .collect(Collectors.toCollection(() -> migrationQueue));
+
+            List<String> closedIndices = migrationQueue.stream()
+                .filter(SystemIndexMigrationInfo::isCurrentIndexClosed)
+                .map(SystemIndexMigrationInfo::getCurrentIndexName)
+                .collect(Collectors.toList());
+            if (closedIndices.isEmpty() == false) {
+                markAsFailed(
+                    new IllegalStateException(
+                        new ParameterizedMessage("indices must be open to be migrated, but indices {} are closed", closedIndices)
+                            .getFormattedMessage()
+                    )
+                );
+                return;
+            }
+
+            // The queue we just generated *should* be the same one as was generated on the last node, so the first entry in the queue
+            // should be the same as is in the task state
+            if (stateIndexName != null && stateFeatureName != null && migrationQueue.isEmpty() == false) {
+                SystemIndexMigrationInfo nextMigrationInfo = migrationQueue.peek();
+                // This should never, ever happen in testing mode, but could conceivably happen if there are different sets of plugins
+                // installed on the previous node vs. this one.
+                assert nextMigrationInfo.getFeatureName().equals(stateFeatureName)
+                    && nextMigrationInfo.getCurrentIndexName().equals(stateIndexName) : "index name ["
+                        + stateIndexName
+                        + "] or feature name ["
+                        + stateFeatureName
+                        + "] from task state did not match first index ["
+                        + nextMigrationInfo.getCurrentIndexName()
+                        + "] and feature ["
+                        + nextMigrationInfo.getFeatureName()
+                        + "] of locally computed queue, see logs";
+                if (nextMigrationInfo.getCurrentIndexName().equals(stateIndexName) == false) {
+                    if (clusterState.metadata().hasIndex(stateIndexName) == false) {
+                        // If we don't have that index at all, and also don't have the next one
+                        markAsFailed(
+                            new IllegalStateException(
+                                new ParameterizedMessage(
+                                    "failed to resume system index migration from index [{}], that index is not present in the cluster",
+                                    stateIndexName
+                                ).getFormattedMessage()
+                            )
+                        );
+                    }
+                    logger.warn(
+                        new ParameterizedMessage(
+                            "resuming system index migration with index [{}], which does not match index given in last task state [{}]",
+                            nextMigrationInfo.getCurrentIndexName(),
+                            stateIndexName
+                        )
+                    );
+                }
+            }
+        }
+
+        // Kick off our callback "loop" - finishIndexAndLoop calls back into prepareNextIndex
+        cleanUpPreviousMigration(
+            taskState,
+            clusterState,
+            state -> prepareNextIndex(state, state2 -> migrateSingleIndex(state2, this::finishIndexAndLoop), stateFeatureName)
+        );
+    }
+
+    private void cleanUpPreviousMigration(
+        SystemIndexMigrationTaskState taskState,
+        ClusterState currentState,
+        Consumer<ClusterState> listener
+    ) {
+        logger.debug("cleaning up previous migration, task state: [{}]", taskState == null ? "null" : Strings.toString(taskState));
+        if (taskState != null && taskState.getCurrentIndex() != null) {
+            SystemIndexMigrationInfo migrationInfo;
+            try {
+                migrationInfo = SystemIndexMigrationInfo.fromTaskState(
+                    taskState,
+                    systemIndices,
+                    currentState.metadata(),
+                    indexScopedSettings
+                );
+            } catch (Exception e) {
+                markAsFailed(e);
+                return;
+            }
+            final String newIndexName = migrationInfo.getNextIndexName();
+            logger.info("removing index [{}] from previous incomplete migration", newIndexName);
+
+            migrationInfo.createClient(baseClient)
+                .admin()
+                .indices()
+                .prepareDelete(newIndexName)
+                .execute(ActionListener.wrap(ackedResponse -> {
+                    if (ackedResponse.isAcknowledged()) {
+                        logger.debug("successfully removed index [{}]", newIndexName);
+                        clearResults(clusterService, ActionListener.wrap(listener::accept, this::markAsFailed));
+                    }
+                }, this::markAsFailed));
+        } else {
+            logger.debug("no incomplete index to remove");
+            clearResults(clusterService, ActionListener.wrap(listener::accept, this::markAsFailed));
+        }
+    }
+
+    private void finishIndexAndLoop(BulkByScrollResponse bulkResponse) {
+        // The BulkByScroll response is validated in #migrateSingleIndex, it's just here to satisfy the ActionListener type
+        assert bulkResponse.isTimedOut() == false
+            && (bulkResponse.getBulkFailures() == null || bulkResponse.getBulkFailures().isEmpty())
+            && (bulkResponse.getSearchFailures() == null
+                || bulkResponse.getSearchFailures()
+                    .isEmpty()) : "If this assertion gets triggered it means the validation in migrateSingleIndex isn't working right";
+        SystemIndexMigrationInfo lastMigrationInfo = currentMigrationInfo();
+        logger.info(
+            "finished migrating old index [{}] from feature [{}] to new index [{}]",
+            lastMigrationInfo.getCurrentIndexName(),
+            lastMigrationInfo.getFeatureName(),
+            lastMigrationInfo.getNextIndexName()
+        );
+        assert migrationQueue != null && migrationQueue.isEmpty() == false;
+        synchronized (migrationQueue) {
+            migrationQueue.remove();
+        }
+        SystemIndexMigrationInfo nextMigrationInfo = currentMigrationInfo();
+        if (nextMigrationInfo == null || nextMigrationInfo.getFeatureName().equals(lastMigrationInfo.getFeatureName()) == false) {
+            // The next feature name is different than the last one, so we just finished a feature - time to invoke its post-migration hook
+            lastMigrationInfo.indicesMigrationComplete(
+                currentFeatureCallbackMetadata.get(),
+                clusterService,
+                baseClient,
+                ActionListener.wrap(successful -> {
+                    if (successful == false) {
+                        // GWB> Should we actually fail in this case instead of plugging along?
+                        logger.warn(
+                            "post-migration hook for feature [{}] indicated failure; feature migration metadata prior to failure was [{}]",
+                            lastMigrationInfo.getFeatureName(),
+                            currentFeatureCallbackMetadata.get()
+                        );
+                    }
+                    recordIndexMigrationSuccess(lastMigrationInfo);
+                }, this::markAsFailed)
+            );
+        } else {
+            prepareNextIndex(
+                clusterService.state(),
+                state2 -> migrateSingleIndex(state2, this::finishIndexAndLoop),
+                lastMigrationInfo.getFeatureName()
+            );
+        }
+    }
+
+    private void recordIndexMigrationSuccess(SystemIndexMigrationInfo lastMigrationInfo) {
+        MigrationResultsUpdateTask updateTask = MigrationResultsUpdateTask.upsert(
+            lastMigrationInfo.getFeatureName(),
+            SingleFeatureMigrationResult.success(),
+            ActionListener.wrap(
+                state -> {
+                    prepareNextIndex(
+                        state,
+                        clusterState -> migrateSingleIndex(clusterState, this::finishIndexAndLoop),
+                        lastMigrationInfo.getFeatureName()
+                    );
+                },
+                this::markAsFailed
+            )
+        );
+        updateTask.submit(clusterService);
+    }
+
+    private void prepareNextIndex(ClusterState clusterState, Consumer<ClusterState> listener, String lastFeatureName) {
+        synchronized (migrationQueue) {
+            assert migrationQueue != null;
+            if (migrationQueue.isEmpty()) {
+                logger.info("finished migrating feature indices");
+                markAsCompleted();
+                return;
+            }
+        }
+
+        final SystemIndexMigrationInfo migrationInfo = currentMigrationInfo();
+        assert migrationInfo != null : "the queue of indices to migrate should have been checked for emptiness before calling this method";
+        logger.info(
+            "preparing to migrate old index [{}] from feature [{}] to new index [{}]",
+            migrationInfo.getCurrentIndexName(),
+            migrationInfo.getFeatureName(),
+            migrationInfo.getNextIndexName()
+        );
+        final AtomicReference<Map<String, Object>> updatedTaskStateFeatureMetadata = new AtomicReference<>();
+        if (migrationInfo.getFeatureName().equals(lastFeatureName) == false) {
+            // And then invoke the pre-migration hook for the next one.
+            migrationInfo.prepareForIndicesMigration(
+                clusterService,
+                baseClient,
+                ActionListener.wrap(updatedTaskStateFeatureMetadata::set, this::markAsFailed)
+            );
+        } else {
+            // Otherwise, just re-use what we already have.
+            updatedTaskStateFeatureMetadata.set(currentFeatureCallbackMetadata.get());
+        }
+        final SystemIndexMigrationTaskState newTaskState = new SystemIndexMigrationTaskState(
+            migrationInfo.getCurrentIndexName(),
+            migrationInfo.getFeatureName(),
+            updatedTaskStateFeatureMetadata.get()
+        );
+        logger.debug("updating task state to [{}]", Strings.toString(newTaskState));
+        currentFeatureCallbackMetadata.set(updatedTaskStateFeatureMetadata.get());
+        updatePersistentTaskState(newTaskState, ActionListener.wrap(task -> {
+            assert newTaskState.equals(task.getState()) : "task state returned by update method did not match submitted task state";
+            logger.debug("new task state [{}] accepted", Strings.toString(newTaskState));
+            listener.accept(clusterState);
+        }, this::markAsFailed));
+    }
+
+    private boolean needsToBeMigrated(IndexMetadata indexMetadata) {
+        assert indexMetadata != null : "null IndexMetadata should be impossible, we're not consistently using the same cluster state";
+        if (indexMetadata == null) {
+            return false;
+        }
+        return indexMetadata.isSystem() && indexMetadata.getCreationVersion().before(NO_UPGRADE_REQUIRED_VERSION);
+    }
+
+    private void migrateSingleIndex(ClusterState clusterState, Consumer<BulkByScrollResponse> listener) {
+        final SystemIndexMigrationInfo migrationInfo = currentMigrationInfo();
+        String oldIndexName = migrationInfo.getCurrentIndexName();
+        final IndexMetadata imd = clusterState.metadata().index(oldIndexName);
+        if (imd.getState().equals(CLOSE)) {
+            logger.error(
+                "unable to migrate index [{}] from feature [{}] because it is closed",
+                oldIndexName,
+                migrationInfo.getFeatureName()
+            );
+            markAsFailed(new IllegalStateException("unable to migrate index [" + oldIndexName + "] because it is closed"));
+            return;
+        }
+        Index oldIndex = imd.getIndex();
+        String newIndexName = migrationInfo.getNextIndexName();
+        logger.info("migrating index [{}] from feature [{}] to new index [{}]", oldIndexName, migrationInfo.getFeatureName(), newIndexName);
+        ActionListener<BulkByScrollResponse> innerListener = ActionListener.wrap(listener::accept, this::markAsFailed);
+        try {
+            Exception versionException = checkNodeVersionsReadyForMigration(clusterState);
+            if (versionException != null) {
+                markAsFailed(versionException);
+                return;
+            }
+            createIndex(migrationInfo, ActionListener.wrap(shardsAcknowledgedResponse -> {
+                logger.debug(
+                    "while migrating [{}] , got create index response: [{}]",
+                    oldIndexName,
+                    Strings.toString(shardsAcknowledgedResponse)
+                );
+                setWriteBlock(
+                    oldIndex,
+                    true,
+                    ActionListener.wrap(setReadOnlyResponse -> reindex(migrationInfo, ActionListener.wrap(bulkByScrollResponse -> {
+                        logger.debug(
+                            "while migrating [{}], got reindex response: [{}]",
+                            oldIndexName,
+                            Strings.toString(bulkByScrollResponse)
+                        );
+                        if ((bulkByScrollResponse.getBulkFailures() != null && bulkByScrollResponse.getBulkFailures().isEmpty() == false)
+                            || (bulkByScrollResponse.getSearchFailures() != null
+                                && bulkByScrollResponse.getSearchFailures().isEmpty() == false)) {
+                            removeReadOnlyBlockOnReindexFailure(
+                                oldIndex,
+                                innerListener,
+                                logAndThrowExceptionForFailures(bulkByScrollResponse)
+                            );
+                        } else {
+                            // Successful completion of reindexing - remove read only and delete old index
+                            setWriteBlock(
+                                oldIndex,
+                                false,
+                                ActionListener.wrap(
+                                    setAliasAndRemoveOldIndex(migrationInfo, bulkByScrollResponse, innerListener),
+                                    innerListener::onFailure
+                                )
+                            );
+                        }
+                    }, e -> {
+                        logger.error(
+                            new ParameterizedMessage(
+                                "error occurred while reindexing index [{}] from feature [{}] to destination index [{}]",
+                                oldIndexName,
+                                migrationInfo.getFeatureName(),
+                                newIndexName
+                            ),
+                            e
+                        );
+                        removeReadOnlyBlockOnReindexFailure(oldIndex, innerListener, e);
+                    })), innerListener::onFailure)
+                );
+            }, innerListener::onFailure));
+        } catch (Exception ex) {
+            logger.error(
+                new ParameterizedMessage(
+                    "error occurred while migrating index [{}] from feature [{}] to new index [{}]",
+                    oldIndexName,
+                    migrationInfo.getFeatureName(),
+                    newIndexName
+                ),
+                ex
+            );
+            removeReadOnlyBlockOnReindexFailure(oldIndex, innerListener, ex);
+            innerListener.onFailure(ex);
+        }
+    }
+
+    private void createIndex(SystemIndexMigrationInfo migrationInfo, ActionListener<ShardsAcknowledgedResponse> listener) {
+        final CreateIndexClusterStateUpdateRequest createRequest = new CreateIndexClusterStateUpdateRequest(
+            "migrate-system-index",
+            migrationInfo.getNextIndexName(),
+            migrationInfo.getNextIndexName()
+        );
+
+        createRequest.waitForActiveShards(ActiveShardCount.ALL)
+            .mappings(migrationInfo.getMappings())
+            .settings(Objects.requireNonNullElse(migrationInfo.getSettings(), Settings.EMPTY));
+        metadataCreateIndexService.createIndex(createRequest, listener);
+    }
+
+    private CheckedConsumer<AcknowledgedResponse, Exception> setAliasAndRemoveOldIndex(
+        SystemIndexMigrationInfo migrationInfo,
+        BulkByScrollResponse bulkByScrollResponse,
+        ActionListener<BulkByScrollResponse> listener
+    ) {
+        final IndicesAliasesRequestBuilder aliasesRequest = migrationInfo.createClient(baseClient).admin().indices().prepareAliases();
+        aliasesRequest.removeIndex(migrationInfo.getCurrentIndexName());
+        aliasesRequest.addAlias(migrationInfo.getNextIndexName(), migrationInfo.getCurrentIndexName());
+
+        // Copy all the aliases from the old index
+        IndexMetadata imd = clusterService.state().metadata().index(migrationInfo.getCurrentIndexName());
+        imd.getAliases().values().forEach(aliasToAdd -> {
+            aliasesRequest.addAliasAction(
+                IndicesAliasesRequest.AliasActions.add()
+                    .index(migrationInfo.getNextIndexName())
+                    .alias(aliasToAdd.alias())
+                    .indexRouting(aliasToAdd.indexRouting())
+                    .searchRouting(aliasToAdd.searchRouting())
+                    .filter(aliasToAdd.filter() == null ? null : aliasToAdd.filter().string())
+                    .writeIndex(null)
+            );
+        });
+
+        // Technically this callback might have a different cluster state, but it shouldn't matter - these indices shouldn't be changing
+        // while we're trying to migrate them.
+        return unsetReadOnlyResponse -> aliasesRequest.execute(
+            ActionListener.wrap(deleteIndexResponse -> listener.onResponse(bulkByScrollResponse), listener::onFailure)
+        );
+    }
+
+    /**
+     * Makes the index readonly if it's not set as a readonly yet
+     */
+    private void setWriteBlock(Index index, boolean readOnlyValue, ActionListener<AcknowledgedResponse> listener) {
+        final Settings readOnlySettings = Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), readOnlyValue).build();
+        UpdateSettingsClusterStateUpdateRequest updateSettingsRequest = new UpdateSettingsClusterStateUpdateRequest().indices(
+            new Index[] { index }
+        ).settings(readOnlySettings).setPreserveExisting(false);
+
+        metadataUpdateSettingsService.updateSettings(updateSettingsRequest, listener);
+    }
+
+    private void reindex(SystemIndexMigrationInfo migrationInfo, ActionListener<BulkByScrollResponse> listener) {
+        ReindexRequest reindexRequest = new ReindexRequest();
+        reindexRequest.setSourceIndices(migrationInfo.getCurrentIndexName());
+        reindexRequest.setDestIndex(migrationInfo.getNextIndexName());
+        reindexRequest.setRefresh(true);
+        migrationInfo.createClient(baseClient).execute(ReindexAction.INSTANCE, reindexRequest, listener);
+    }
+
+    // Failure handlers
+    private void removeReadOnlyBlockOnReindexFailure(Index index, ActionListener<BulkByScrollResponse> listener, Exception ex) {
+        logger.info("removing read only block on [{}] because reindex failed [{}]", index, ex);
+        setWriteBlock(index, false, ActionListener.wrap(unsetReadOnlyResponse -> listener.onFailure(ex), e1 -> listener.onFailure(ex)));
+    }
+
+    private ElasticsearchException logAndThrowExceptionForFailures(BulkByScrollResponse bulkByScrollResponse) {
+        String bulkFailures = (bulkByScrollResponse.getBulkFailures() != null)
+            ? Strings.collectionToCommaDelimitedString(bulkByScrollResponse.getBulkFailures())
+            : "";
+        String searchFailures = (bulkByScrollResponse.getSearchFailures() != null)
+            ? Strings.collectionToCommaDelimitedString(bulkByScrollResponse.getSearchFailures())
+            : "";
+        logger.error("error occurred while reindexing, bulk failures [{}], search failures [{}]", bulkFailures, searchFailures);
+        return new ElasticsearchException(
+            "error occurred while reindexing, bulk failures [{}], search failures [{}]",
+            bulkFailures,
+            searchFailures
+        );
+    }
+
+    /**
+     * Clears the migration queue, marks the task as failed, and saves the exception for later retrieval.
+     *
+     * This method is **ASYNC**, so if you're not using it as a listener, be sure to {@code return} after calling this.
+     */
+    @Override
+    public void markAsFailed(Exception e) {
+        SystemIndexMigrationInfo migrationInfo = currentMigrationInfo();
+        synchronized (migrationQueue) {
+            migrationQueue.clear();
+        }
+        String featureName = Optional.ofNullable(migrationInfo).map(SystemIndexMigrationInfo::getFeatureName).orElse("<unknown feature>");
+        String indexName = Optional.ofNullable(migrationInfo).map(SystemIndexMigrationInfo::getCurrentIndexName).orElse("<unknown index>");
+
+        MigrationResultsUpdateTask.upsert(
+            featureName,
+            SingleFeatureMigrationResult.failure(indexName, e),
+            ActionListener.wrap(state -> super.markAsFailed(e), exception -> super.markAsFailed(e))
+        ).submit(clusterService);
+        super.markAsFailed(e);
+    }
+
+    private static Exception checkNodeVersionsReadyForMigration(ClusterState state) {
+        final Version minNodeVersion = state.nodes().getMinNodeVersion();
+        if (minNodeVersion.before(READY_FOR_MIGRATION_VERSION)) {
+            return new IllegalStateException(
+                "all nodes must be on version ["
+                    + READY_FOR_MIGRATION_VERSION
+                    + "] or later to migrate feature indices but lowest node version currently in cluster is ["
+                    + minNodeVersion
+                    + "]"
+            );
+        }
+        return null;
+    }
+
+    /**
+     * Creates a task that will clear the results of previous migration attempts.
+     * @param clusterService The cluster service.
+     * @param listener A listener that will be called upon successfully updating the cluster state.
+     */
+    private static void clearResults(ClusterService clusterService, ActionListener<ClusterState> listener) {
+        clusterService.submitStateUpdateTask("clear migration results", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                if (currentState.metadata().custom(FeatureMigrationResults.TYPE) != null) {
+                    return ClusterState.builder(currentState)
+                        .metadata(Metadata.builder(currentState.metadata()).removeCustom(FeatureMigrationResults.TYPE))
+                        .build();
+                }
+                return currentState;
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                listener.onResponse(newState);
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                logger.error("failed to clear migration results when starting new migration", e);
+                listener.onFailure(e);
+            }
+        });
+        logger.debug("submitted update task to clear migration results");
+    }
+
+    private SystemIndexMigrationInfo currentMigrationInfo() {
+        synchronized (migrationQueue) {
+            return migrationQueue.peek();
+        }
+    }
+}
diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy
index 85f36cf300b35..5ae15e74ec2d4 100644
--- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -22,8 +22,6 @@ grant codeBase "${codebase.elasticsearch-secure-sm}" {
 grant codeBase "${codebase.elasticsearch}" {
   // needed for loading plugins which may expect the context class loader to be set
   permission java.lang.RuntimePermission "setContextClassLoader";
-  // needed by HotThreads to enable wait/block time accounting on demand
-  permission java.lang.management.ManagementPermission "control";
 };
 
 //// Very special jar permissions:
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java
index fd3a63dd290a1..64f10b9e4bcfa 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.health.ClusterIndexHealthTests;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -43,7 +44,7 @@ public class ClusterHealthResponsesTests extends AbstractSerializingTestCase<Clu
     private final ClusterHealthRequest.Level level = randomFrom(ClusterHealthRequest.Level.values());
 
     public void testIsTimeout() {
-        ClusterHealthResponse res = new ClusterHealthResponse("", new String[]{}, ClusterState.EMPTY_STATE, false);
+        ClusterHealthResponse res = new ClusterHealthResponse("", Strings.EMPTY_ARRAY, ClusterState.EMPTY_STATE, false);
         for (int i = 0; i < 5; i++) {
             res.setTimedOut(randomBoolean());
             if (res.isTimedOut()) {
@@ -56,7 +57,7 @@ public void testIsTimeout() {
     }
 
     public void testTimeoutReturns200IfOptedIn() {
-        ClusterHealthResponse res = new ClusterHealthResponse("", new String[]{}, ClusterState.EMPTY_STATE, true);
+        ClusterHealthResponse res = new ClusterHealthResponse("", Strings.EMPTY_ARRAY, ClusterState.EMPTY_STATE, true);
         for (int i = 0; i < 5; i++) {
             res.setTimedOut(randomBoolean());
             assertEquals(RestStatus.OK, res.status());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/hotthreads/NodesHotThreadsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/hotthreads/NodesHotThreadsRequestTests.java
index 590ce0abc4522..645bb3352f26d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/hotthreads/NodesHotThreadsRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/hotthreads/NodesHotThreadsRequestTests.java
@@ -55,7 +55,7 @@ public void testBWCSerialization() throws IOException {
             out.setVersion(previous);
             request.writeTo(out);
             try (StreamInput in = out.bytes().streamInput()) {
-                in.setVersion(latest);
+                in.setVersion(previous);
                 NodesHotThreadsRequest deserialized = new NodesHotThreadsRequest(in);
                 assertEquals(request.threads(), deserialized.threads());
                 assertEquals(request.ignoreIdleThreads(), deserialized.ignoreIdleThreads());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java
index 7966c2171ee87..c4881612edbcc 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java
@@ -15,7 +15,10 @@
 import java.io.IOException;
 import java.util.Collections;
 
-import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.UPGRADE_NEEDED;
+import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR;
+import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS;
+import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED;
+import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -50,24 +53,49 @@ protected GetFeatureUpgradeStatusResponse mutateInstance(GetFeatureUpgradeStatus
 
     /** If constructor is called with null for a list, we just use an empty list */
     public void testConstructorHandlesNullLists() {
-        GetFeatureUpgradeStatusResponse response = new GetFeatureUpgradeStatusResponse(null, UPGRADE_NEEDED);
+        GetFeatureUpgradeStatusResponse response = new GetFeatureUpgradeStatusResponse(null, MIGRATION_NEEDED);
         assertThat(response.getFeatureUpgradeStatuses(), notNullValue());
         assertThat(response.getFeatureUpgradeStatuses(), equalTo(Collections.emptyList()));
     }
 
+    public void testUpgradeStatusCominations() {
+        assertEquals(NO_MIGRATION_NEEDED, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(NO_MIGRATION_NEEDED, NO_MIGRATION_NEEDED));
+
+        assertEquals(MIGRATION_NEEDED, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(NO_MIGRATION_NEEDED, MIGRATION_NEEDED));
+        assertEquals(MIGRATION_NEEDED, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(MIGRATION_NEEDED, NO_MIGRATION_NEEDED));
+        assertEquals(MIGRATION_NEEDED, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(MIGRATION_NEEDED, MIGRATION_NEEDED));
+
+
+        assertEquals(IN_PROGRESS, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(IN_PROGRESS, NO_MIGRATION_NEEDED));
+        assertEquals(IN_PROGRESS, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(NO_MIGRATION_NEEDED, IN_PROGRESS));
+        assertEquals(IN_PROGRESS, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(MIGRATION_NEEDED, IN_PROGRESS));
+        assertEquals(IN_PROGRESS, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(IN_PROGRESS, MIGRATION_NEEDED));
+        assertEquals(IN_PROGRESS, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(IN_PROGRESS, IN_PROGRESS));
+
+
+        assertEquals(ERROR, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(ERROR, NO_MIGRATION_NEEDED));
+        assertEquals(ERROR, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(NO_MIGRATION_NEEDED, ERROR));
+        assertEquals(ERROR, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(MIGRATION_NEEDED, ERROR));
+        assertEquals(ERROR, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(ERROR, MIGRATION_NEEDED));
+        assertEquals(ERROR, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(IN_PROGRESS, ERROR));
+        assertEquals(ERROR, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(ERROR, IN_PROGRESS));
+        assertEquals(ERROR, GetFeatureUpgradeStatusResponse.UpgradeStatus.combine(ERROR, ERROR));
+    }
+
     private static GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus createFeatureStatus() {
         return new GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus(
             randomAlphaOfLengthBetween(3, 20),
             randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()),
             randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()),
-            randomList(4, GetFeatureUpgradeStatusResponseTests::getIndexVersion)
+            randomList(4, GetFeatureUpgradeStatusResponseTests::getIndexInfo)
         );
     }
 
-    private static GetFeatureUpgradeStatusResponse.IndexVersion getIndexVersion() {
-        return new GetFeatureUpgradeStatusResponse.IndexVersion(
+    private static GetFeatureUpgradeStatusResponse.IndexInfo getIndexInfo() {
+        return new GetFeatureUpgradeStatusResponse.IndexInfo(
             randomAlphaOfLengthBetween(3, 20),
-            randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion())
+            randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()),
+            null
         );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java
index 5fb9cba2b4b18..db2b2f3d09a62 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java
@@ -20,9 +20,8 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 
-import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_UPGRADE_NEEDED;
+import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED;
 import static org.hamcrest.Matchers.equalTo;
 
 public class TransportGetFeatureUpgradeStatusActionTests extends ESTestCase {
@@ -35,27 +34,27 @@ public void testGetFeatureStatus() {
         GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus status =
             TransportGetFeatureUpgradeStatusAction.getFeatureUpgradeStatus(
                 CLUSTER_STATE,
-                Map.entry("test-feature", FEATURE));
+                FEATURE);
 
-        assertThat(status.getUpgradeStatus(), equalTo(NO_UPGRADE_NEEDED));
+        assertThat(status.getUpgradeStatus(), equalTo(MIGRATION_NEEDED));
         assertThat(status.getFeatureName(), equalTo("test-feature"));
         assertThat(status.getMinimumIndexVersion(), equalTo(Version.V_7_0_0));
         assertThat(status.getIndexVersions().size(), equalTo(2)); // additional testing below
     }
 
-    public void testGetIndexVersion() {
-        List<GetFeatureUpgradeStatusResponse.IndexVersion> versions =
-            TransportGetFeatureUpgradeStatusAction.getIndexVersions(CLUSTER_STATE, FEATURE);
+    public void testGetIndexInfos() {
+        List<GetFeatureUpgradeStatusResponse.IndexInfo> versions =
+            TransportGetFeatureUpgradeStatusAction.getIndexInfos(CLUSTER_STATE, FEATURE);
 
         assertThat(versions.size(), equalTo(2));
 
         {
-            GetFeatureUpgradeStatusResponse.IndexVersion version = versions.get(0);
+            GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(0);
             assertThat(version.getVersion(), equalTo(Version.CURRENT));
             assertThat(version.getIndexName(), equalTo(".test-index-1"));
         }
         {
-            GetFeatureUpgradeStatusResponse.IndexVersion version = versions.get(1);
+            GetFeatureUpgradeStatusResponse.IndexInfo version = versions.get(1);
             assertThat(version.getVersion(), equalTo(Version.V_7_0_0));
             assertThat(version.getIndexName(), equalTo(".test-index-2"));
         }
diff --git a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
index 8e9f19a8b418f..19258b6f9c65c 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
@@ -44,7 +44,7 @@ public void testRamBytesUsed() throws Exception {
                 map.putIndexUnderLock(uid.toBytesRef(), randomIndexVersionValue());
             }
         }
-        long actualRamBytesUsed = RamUsageTester.sizeOf(map);
+        long actualRamBytesUsed = RamUsageTester.ramUsed(map);
         long estimatedRamBytesUsed = map.ramBytesUsed();
         // less than 50% off
         assertEquals(actualRamBytesUsed, estimatedRamBytesUsed, actualRamBytesUsed / 2);
@@ -60,7 +60,7 @@ public void testRamBytesUsed() throws Exception {
                 map.putIndexUnderLock(uid.toBytesRef(), randomIndexVersionValue());
             }
         }
-        actualRamBytesUsed = RamUsageTester.sizeOf(map);
+        actualRamBytesUsed = RamUsageTester.ramUsed(map);
         estimatedRamBytesUsed = map.ramBytesUsed();
         long tolerance;
         if (Constants.JRE_IS_MINIMUM_JAVA9) {
diff --git a/server/src/test/java/org/elasticsearch/index/engine/VersionValueTests.java b/server/src/test/java/org/elasticsearch/index/engine/VersionValueTests.java
index dfb41dd18c0ad..7f9c3ab155bab 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/VersionValueTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/VersionValueTests.java
@@ -20,12 +20,12 @@ public void testIndexRamBytesUsed() {
             translogLoc = new Translog.Location(randomNonNegativeLong(), randomNonNegativeLong(), randomInt());
         }
         IndexVersionValue versionValue = new IndexVersionValue(translogLoc, randomLong(), randomLong(), randomLong());
-        assertEquals(RamUsageTester.sizeOf(versionValue), versionValue.ramBytesUsed());
+        assertEquals(RamUsageTester.ramUsed(versionValue), versionValue.ramBytesUsed());
     }
 
     public void testDeleteRamBytesUsed() {
         DeleteVersionValue versionValue = new DeleteVersionValue(randomLong(), randomLong(), randomLong(), randomLong());
-        assertEquals(RamUsageTester.sizeOf(versionValue), versionValue.ramBytesUsed());
+        assertEquals(RamUsageTester.ramUsed(versionValue), versionValue.ramBytesUsed());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index a8cc3bace932e..4072b15018fb9 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.logging.log4j.Level;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
@@ -148,9 +149,9 @@ public void testResolutionLossDeprecation() throws Exception {
 
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", "2018-10-03T14:42:44.123456+0000")));
 
-        assertWarnings("You are attempting to store a nanosecond resolution " +
+        assertWarnings(true, new DeprecationWarning(Level.WARN, "You are attempting to store a nanosecond resolution " +
             "on a field [field] of type date on index [index]. " +
-            "The nanosecond part was lost. Use date_nanos field type.");
+            "The nanosecond part was lost. Use date_nanos field type."));
     }
 
     private void testIgnoreMalformedForValue(String value, String expectedCause, String dateFormat) throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/monitor/jvm/HotThreadsTests.java b/server/src/test/java/org/elasticsearch/monitor/jvm/HotThreadsTests.java
index 68f536716f926..255fc09f1dbdf 100644
--- a/server/src/test/java/org/elasticsearch/monitor/jvm/HotThreadsTests.java
+++ b/server/src/test/java/org/elasticsearch/monitor/jvm/HotThreadsTests.java
@@ -24,6 +24,7 @@
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.stringContainsInOrder;
 import static org.mockito.Matchers.anyInt;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.inOrder;
@@ -32,6 +33,10 @@
 
 public class HotThreadsTests extends ESTestCase {
 
+    private long nanosHelper(long millis) {
+        return millis * 1_000_000;
+    }
+
     public void testSupportedThreadsReportType() {
         for (String type : new String[]{"unsupported", "", null, "CPU", "WAIT", "BLOCK", "MEM"}) {
             expectThrows(IllegalArgumentException.class, () -> new HotThreads().type(HotThreads.ReportType.of(type)));
@@ -60,7 +65,8 @@ private List<StackTraceElement> makeThreadStackHelper(List<String[]> names) {
 
     public void testIdleThreadsDetection() {
         for (String threadName : new String[]{
-            "Signal Dispatcher", "Finalizer", "Reference Handler", "Notification Thread", "Common-Cleaner", "process reaper"}) {
+            "Signal Dispatcher", "Finalizer", "Reference Handler", "Notification Thread",
+            "Common-Cleaner", "process reaper", "DestroyJavaVM"}) {
             ThreadInfo mockedThreadInfo = mock(ThreadInfo.class);
             when(mockedThreadInfo.getThreadName()).thenReturn(threadName);
             assertTrue(HotThreads.isKnownJDKThread(mockedThreadInfo));
@@ -196,7 +202,11 @@ class SimilarityTestCase {
     }
 
     private ThreadInfo makeThreadInfoMocksHelper(ThreadMXBean mockedMXBean, long threadId) {
-        when(mockedMXBean.getThreadCpuTime(threadId)).thenReturn(0L).thenReturn(threadId);
+        return makeThreadInfoMocksHelper(mockedMXBean, threadId, 1L);
+    }
+
+    private ThreadInfo makeThreadInfoMocksHelper(ThreadMXBean mockedMXBean, long threadId, long cpuMultiplier) {
+        when(mockedMXBean.getThreadCpuTime(threadId)).thenReturn(0L).thenReturn(threadId * cpuMultiplier);
         ThreadInfo mockedThreadInfo = mock(ThreadInfo.class);
         when(mockedMXBean.getThreadInfo(eq(threadId), anyInt())).thenReturn(mockedThreadInfo);
         when(mockedThreadInfo.getThreadName()).thenReturn(String.format(Locale.ROOT, "Thread %d", threadId));
@@ -222,12 +232,16 @@ private ThreadInfo makeThreadInfoMocksHelper(ThreadMXBean mockedMXBean, long thr
         return mockedThreadInfo;
     }
 
-    // We call this helper for each different mode to reset the before and after timings.
     private List<ThreadInfo> makeThreadInfoMocksHelper(ThreadMXBean mockedMXBean, long[] threadIds) {
+        return makeThreadInfoMocksHelper(mockedMXBean, threadIds, 1L);
+    }
+
+    // We call this helper for each different mode to reset the before and after timings.
+    private List<ThreadInfo> makeThreadInfoMocksHelper(ThreadMXBean mockedMXBean, long[] threadIds, long cpuMultiplier) {
         List<ThreadInfo> allInfos = new ArrayList<>(threadIds.length);
 
         for (long threadId : threadIds) {
-            allInfos.add(makeThreadInfoMocksHelper(mockedMXBean, threadId));
+            allInfos.add(makeThreadInfoMocksHelper(mockedMXBean, threadId, cpuMultiplier));
         }
 
         when(mockedMXBean.getThreadInfo(Matchers.any(), anyInt())).thenReturn(allInfos.toArray(new ThreadInfo[0]));
@@ -236,110 +250,222 @@ private List<ThreadInfo> makeThreadInfoMocksHelper(ThreadMXBean mockedMXBean, lo
         return allInfos;
     }
 
-    public void testInnerDetect() throws Exception {
+    private ThreadMXBean makeMockMXBeanHelper() {
         ThreadMXBean mockedMXBean = mock(ThreadMXBean.class);
         when(mockedMXBean.isThreadCpuTimeSupported()).thenReturn(true);
+        when(mockedMXBean.isThreadContentionMonitoringSupported()).thenReturn(true);
+        when(mockedMXBean.isThreadContentionMonitoringEnabled()).thenReturn(true);
 
+        return mockedMXBean;
+    }
+
+    private SunThreadInfo makeMockSunThreadInfoHelper() {
         SunThreadInfo mockedSunThreadInfo = mock(SunThreadInfo.class);
         when(mockedSunThreadInfo.isThreadAllocatedMemorySupported()).thenReturn(true);
         when(mockedSunThreadInfo.isThreadAllocatedMemoryEnabled()).thenReturn(true);
 
+        return mockedSunThreadInfo;
+    }
+
+    public void testInnerDetectCPUMode() throws Exception {
+        ThreadMXBean mockedMXBean = makeMockMXBeanHelper();
+        SunThreadInfo mockedSunThreadInfo = makeMockSunThreadInfoHelper();
+
         long[] threadIds = new long[]{1, 2, 3, 4}; // Adds up to 10, the intervalNanos for calculating time percentages
         long mockCurrentThreadId = 0L;
         when(mockedMXBean.getAllThreadIds()).thenReturn(threadIds);
 
-        List<ThreadInfo> allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds);
-        List<ThreadInfo> cpuOrderedInfos = List.of(allInfos.get(3), allInfos.get(2), allInfos.get(1), allInfos.get(0));
+        List<ThreadInfo> allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds, 100_000);
+        List<ThreadInfo> cpuOrderedInfos = List.of(allInfos.get(0), allInfos.get(1), allInfos.get(2), allInfos.get(3));
         when(mockedMXBean.getThreadInfo(Matchers.any(), anyInt())).thenReturn(cpuOrderedInfos.toArray(new ThreadInfo[0]));
 
         HotThreads hotThreads = new HotThreads()
             .busiestThreads(4)
             .type(HotThreads.ReportType.CPU)
-            .interval(TimeValue.timeValueNanos(10))
+            .interval(TimeValue.timeValueMillis(10))
             .threadElementsSnapshotCount(11)
             .ignoreIdleThreads(false);
 
-        String innerResult = hotThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId);
+        String innerResult = hotThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId, (interval) -> null);
 
         assertThat(innerResult, containsString("Hot threads at "));
-        assertThat(innerResult, containsString("interval=10nanos, busiestThreads=4, ignoreIdleThreads=false:"));
+        assertThat(innerResult, containsString("interval=10ms, busiestThreads=4, ignoreIdleThreads=false:"));
         assertThat(innerResult, containsString("11/11 snapshots sharing following 2 elements"));
-        assertThat(innerResult, containsString("40.0% (4nanos out of 10nanos) cpu usage by thread 'Thread 4'"));
-        assertThat(innerResult, containsString("30.0% (3nanos out of 10nanos) cpu usage by thread 'Thread 3'"));
-        assertThat(innerResult, containsString("20.0% (2nanos out of 10nanos) cpu usage by thread 'Thread 2'"));
-        assertThat(innerResult, containsString("10.0% (1nanos out of 10nanos) cpu usage by thread 'Thread 1'"));
+        assertThat(innerResult, stringContainsInOrder(
+            "90.0% [cpu=1.0%, other=89.0%] (9ms out of 10ms) cpu usage by thread 'Thread 1'",
+            "80.0% [cpu=2.0%, other=78.0%] (8ms out of 10ms) cpu usage by thread 'Thread 2'",
+            "70.0% [cpu=3.0%, other=67.0%] (7ms out of 10ms) cpu usage by thread 'Thread 3'",
+            "60.0% [cpu=4.0%, other=56.0%] (6ms out of 10ms) cpu usage by thread 'Thread 4'"));
         assertThat(innerResult, containsString("org.elasticsearch.monitor.test.method_0(Some_File:1)"));
         assertThat(innerResult, containsString("org.elasticsearch.monitor.test.method_1(Some_File:1)"));
         assertThat(innerResult, containsString("org.elasticsearch.monitor.testOther.methodFinal(Some_File:1)"));
 
+
         // Let's ask again without progressing the CPU thread counters, e.g. resetting the mocks
-        innerResult = hotThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId);
+        innerResult = hotThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId, (interval) -> null);
+
+        assertThat(innerResult, containsString("0.0% [cpu=0.0%, other=0.0%] (0s out of 10ms) cpu usage by thread 'Thread 4'"));
+        assertThat(innerResult, containsString("0.0% [cpu=0.0%, other=0.0%] (0s out of 10ms) cpu usage by thread 'Thread 3'"));
+        assertThat(innerResult, containsString("0.0% [cpu=0.0%, other=0.0%] (0s out of 10ms) cpu usage by thread 'Thread 2'"));
+        assertThat(innerResult, containsString("0.0% [cpu=0.0%, other=0.0%] (0s out of 10ms) cpu usage by thread 'Thread 1'"));
 
-        assertThat(innerResult, containsString("0.0% (0s out of 10nanos) cpu usage by thread 'Thread 4'"));
-        assertThat(innerResult, containsString("0.0% (0s out of 10nanos) cpu usage by thread 'Thread 3'"));
-        assertThat(innerResult, containsString("0.0% (0s out of 10nanos) cpu usage by thread 'Thread 2'"));
-        assertThat(innerResult, containsString("0.0% (0s out of 10nanos) cpu usage by thread 'Thread 1'"));
+        // Test with the legacy sort order
+        allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds, 100_000);
+        cpuOrderedInfos = List.of(allInfos.get(3), allInfos.get(2), allInfos.get(1), allInfos.get(0));
+        when(mockedMXBean.getThreadInfo(Matchers.any(), anyInt())).thenReturn(cpuOrderedInfos.toArray(new ThreadInfo[0]));
+
+        hotThreads = new HotThreads()
+            .busiestThreads(4)
+            .type(HotThreads.ReportType.CPU)
+            .interval(TimeValue.timeValueMillis(10))
+            .sortOrder(HotThreads.SortOrder.CPU)
+            .threadElementsSnapshotCount(11)
+            .ignoreIdleThreads(false);
+
+        innerResult = hotThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId, (interval) -> null);
+
+        assertThat(innerResult, containsString("Hot threads at "));
+        assertThat(innerResult, containsString("interval=10ms, busiestThreads=4, ignoreIdleThreads=false:"));
+        assertThat(innerResult, containsString("11/11 snapshots sharing following 2 elements"));
+        assertThat(innerResult, stringContainsInOrder(
+            "60.0% [cpu=4.0%, other=56.0%] (6ms out of 10ms) cpu usage by thread 'Thread 4'",
+            "70.0% [cpu=3.0%, other=67.0%] (7ms out of 10ms) cpu usage by thread 'Thread 3'",
+            "80.0% [cpu=2.0%, other=78.0%] (8ms out of 10ms) cpu usage by thread 'Thread 2'",
+            "90.0% [cpu=1.0%, other=89.0%] (9ms out of 10ms) cpu usage by thread 'Thread 1'"));
+        assertThat(innerResult, containsString("org.elasticsearch.monitor.test.method_0(Some_File:1)"));
+        assertThat(innerResult, containsString("org.elasticsearch.monitor.test.method_1(Some_File:1)"));
+        assertThat(innerResult, containsString("org.elasticsearch.monitor.testOther.methodFinal(Some_File:1)"));
+    }
+
+    public void testInnerDetectWaitMode() throws Exception {
+        ThreadMXBean mockedMXBean = makeMockMXBeanHelper();
+        SunThreadInfo mockedSunThreadInfo = makeMockSunThreadInfoHelper();
+
+        long[] threadIds = new long[]{1, 2, 3, 4}; // Adds up to 10, the intervalNanos for calculating time percentages
+        long mockCurrentThreadId = 0L;
+        when(mockedMXBean.getAllThreadIds()).thenReturn(threadIds);
 
         HotThreads hotWaitingThreads = new HotThreads()
             .busiestThreads(4)
             .type(HotThreads.ReportType.WAIT)
-            .interval(TimeValue.timeValueNanos(10))
+            .interval(TimeValue.timeValueMillis(50))
             .threadElementsSnapshotCount(11)
             .ignoreIdleThreads(false);
 
-        allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds);
+        List<ThreadInfo> allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds);
         List<ThreadInfo> waitOrderedInfos = List.of(allInfos.get(3), allInfos.get(1), allInfos.get(0), allInfos.get(2));
         when(mockedMXBean.getThreadInfo(Matchers.any(), anyInt())).thenReturn(waitOrderedInfos.toArray(new ThreadInfo[0]));
 
-        String waitInnerResult = hotWaitingThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId);
+        String waitInnerResult = hotWaitingThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId, (interval) -> null);
 
-        assertThat(waitInnerResult, containsString("40.0% (4nanos out of 10nanos) wait usage by thread 'Thread 4'"));
-        assertThat(waitInnerResult, containsString("20.0% (2nanos out of 10nanos) wait usage by thread 'Thread 2'"));
-        assertThat(waitInnerResult, containsString("0.0% (0s out of 10nanos) wait usage by thread 'Thread 1'"));
-        assertThat(waitInnerResult, containsString("0.0% (0s out of 10nanos) wait usage by thread 'Thread 3'"));
+        assertThat(waitInnerResult, stringContainsInOrder(
+            "8.0% (4ms out of 50ms) wait usage by thread 'Thread 4'",
+            "4.0% (2ms out of 50ms) wait usage by thread 'Thread 2'",
+            "0.0% (0s out of 50ms) wait usage by thread 'Thread 1'",
+            "0.0% (0s out of 50ms) wait usage by thread 'Thread 3'"));
+
+        // Sort order has no impact on wait mode
+        hotWaitingThreads = new HotThreads()
+            .busiestThreads(4)
+            .type(HotThreads.ReportType.WAIT)
+            .sortOrder(HotThreads.SortOrder.CPU)
+            .interval(TimeValue.timeValueMillis(50))
+            .threadElementsSnapshotCount(11)
+            .ignoreIdleThreads(false);
 
+        allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds);
+        waitOrderedInfos = List.of(allInfos.get(3), allInfos.get(1), allInfos.get(0), allInfos.get(2));
+        when(mockedMXBean.getThreadInfo(Matchers.any(), anyInt())).thenReturn(waitOrderedInfos.toArray(new ThreadInfo[0]));
+
+        waitInnerResult = hotWaitingThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId, (interval) -> null);
+
+        assertThat(waitInnerResult, stringContainsInOrder(
+            "8.0% (4ms out of 50ms) wait usage by thread 'Thread 4'",
+            "4.0% (2ms out of 50ms) wait usage by thread 'Thread 2'",
+            "0.0% (0s out of 50ms) wait usage by thread 'Thread 1'",
+            "0.0% (0s out of 50ms) wait usage by thread 'Thread 3'"));
+    }
+
+    public void testInnerDetectBlockedMode() throws Exception {
+        ThreadMXBean mockedMXBean = makeMockMXBeanHelper();
+        SunThreadInfo mockedSunThreadInfo = makeMockSunThreadInfoHelper();
+
+        long[] threadIds = new long[]{1, 2, 3, 4}; // Adds up to 10, the intervalNanos for calculating time percentages
+        long mockCurrentThreadId = 0L;
+        when(mockedMXBean.getAllThreadIds()).thenReturn(threadIds);
         HotThreads hotBlockedThreads = new HotThreads()
             .busiestThreads(4)
             .type(HotThreads.ReportType.BLOCK)
-            .interval(TimeValue.timeValueNanos(10))
+            .interval(TimeValue.timeValueMillis(60))
             .threadElementsSnapshotCount(11)
             .ignoreIdleThreads(false);
 
-        allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds);
+        List<ThreadInfo> allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds);
         List<ThreadInfo> blockOrderedInfos = List.of(allInfos.get(2), allInfos.get(0), allInfos.get(1), allInfos.get(3));
         when(mockedMXBean.getThreadInfo(Matchers.any(), anyInt())).thenReturn(blockOrderedInfos.toArray(new ThreadInfo[0]));
 
-        String blockInnerResult = hotBlockedThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId);
+        String blockInnerResult = hotBlockedThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId, (interval) -> null);
 
-        assertThat(blockInnerResult, containsString("30.0% (3nanos out of 10nanos) block usage by thread 'Thread 3'"));
-        assertThat(blockInnerResult, containsString("10.0% (1nanos out of 10nanos) block usage by thread 'Thread 1'"));
-        assertThat(blockInnerResult, containsString("0.0% (0s out of 10nanos) block usage by thread 'Thread 2'"));
-        assertThat(blockInnerResult, containsString("0.0% (0s out of 10nanos) block usage by thread 'Thread 4'"));
+        assertThat(blockInnerResult, stringContainsInOrder(
+            "5.0% (3ms out of 60ms) block usage by thread 'Thread 3'",
+            "1.7% (1ms out of 60ms) block usage by thread 'Thread 1'",
+            "0.0% (0s out of 60ms) block usage by thread 'Thread 2'",
+            "0.0% (0s out of 60ms) block usage by thread 'Thread 4'"));
 
-        // Test with only one stack to trigger the different print in innerDetect
+        // Sort order has no impact on block mode
+        hotBlockedThreads = new HotThreads()
+            .busiestThreads(4)
+            .type(HotThreads.ReportType.BLOCK)
+            .sortOrder(HotThreads.SortOrder.CPU)
+            .interval(TimeValue.timeValueMillis(60))
+            .threadElementsSnapshotCount(11)
+            .ignoreIdleThreads(false);
 
         allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds);
-        cpuOrderedInfos = List.of(allInfos.get(3), allInfos.get(2), allInfos.get(1), allInfos.get(0));
+        blockOrderedInfos = List.of(allInfos.get(2), allInfos.get(0), allInfos.get(1), allInfos.get(3));
+        when(mockedMXBean.getThreadInfo(Matchers.any(), anyInt())).thenReturn(blockOrderedInfos.toArray(new ThreadInfo[0]));
+
+        blockInnerResult = hotBlockedThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId, (interval) -> null);
+
+        assertThat(blockInnerResult, stringContainsInOrder(
+            "5.0% (3ms out of 60ms) block usage by thread 'Thread 3'",
+            "1.7% (1ms out of 60ms) block usage by thread 'Thread 1'",
+            "0.0% (0s out of 60ms) block usage by thread 'Thread 2'",
+            "0.0% (0s out of 60ms) block usage by thread 'Thread 4'"));
+    }
+
+    public void testInnerDetectMemoryMode() throws Exception {
+        ThreadMXBean mockedMXBean = makeMockMXBeanHelper();
+        SunThreadInfo mockedSunThreadInfo = makeMockSunThreadInfoHelper();
+
+        long[] threadIds = new long[]{1, 2, 3, 4}; // Adds up to 10, the intervalNanos for calculating time percentages
+        long mockCurrentThreadId = 0L;
+        when(mockedMXBean.getAllThreadIds()).thenReturn(threadIds);
+
+        List<ThreadInfo> allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds);
+        List<ThreadInfo> cpuOrderedInfos = List.of(allInfos.get(3), allInfos.get(2), allInfos.get(1), allInfos.get(0));
         when(mockedMXBean.getThreadInfo(Matchers.any(), anyInt())).thenReturn(cpuOrderedInfos.toArray(new ThreadInfo[0]));
 
-        hotThreads = new HotThreads()
+        for (long threadId : threadIds) {
+            when(mockedSunThreadInfo.getThreadAllocatedBytes(threadId)).thenReturn(0L).thenReturn(threadId*100);
+        }
+
+        HotThreads hotThreads = new HotThreads()
             .busiestThreads(4)
-            .type(HotThreads.ReportType.CPU)
+            .type(HotThreads.ReportType.MEM)
             .interval(TimeValue.timeValueNanos(10))
             .threadElementsSnapshotCount(1)
             .ignoreIdleThreads(false);
 
-        String singleResult = hotThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId);
+        String memInnerResult = hotThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId, (interval) -> null);
+        assertThat(memInnerResult, containsString("  unique snapshot"));
+        assertThat(memInnerResult, stringContainsInOrder(
+            "400b memory allocated by thread 'Thread 4'",
+            "300b memory allocated by thread 'Thread 3'",
+            "200b memory allocated by thread 'Thread 2'",
+            "100b memory allocated by thread 'Thread 1'"));
 
-        assertThat(singleResult, containsString("  unique snapshot"));
-        assertEquals(5, singleResult.split(" unique snapshot").length);
-        assertThat(singleResult, containsString("40.0% (4nanos out of 10nanos) cpu usage by thread 'Thread 4'"));
-        assertThat(singleResult, containsString("30.0% (3nanos out of 10nanos) cpu usage by thread 'Thread 3'"));
-        assertThat(singleResult, containsString("20.0% (2nanos out of 10nanos) cpu usage by thread 'Thread 2'"));
-        assertThat(singleResult, containsString("10.0% (1nanos out of 10nanos) cpu usage by thread 'Thread 1'"));
-        assertThat(innerResult, containsString("org.elasticsearch.monitor.test.method_0(Some_File:1)"));
-        assertThat(innerResult, containsString("org.elasticsearch.monitor.test.method_1(Some_File:1)"));
-        assertThat(innerResult, containsString("org.elasticsearch.monitor.testOther.methodFinal(Some_File:1)"));
+        // Sort order has no impact on memory mode
 
         allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds);
         cpuOrderedInfos = List.of(allInfos.get(3), allInfos.get(2), allInfos.get(1), allInfos.get(0));
@@ -352,21 +478,55 @@ public void testInnerDetect() throws Exception {
         hotThreads = new HotThreads()
             .busiestThreads(4)
             .type(HotThreads.ReportType.MEM)
+            .sortOrder(HotThreads.SortOrder.CPU)
             .interval(TimeValue.timeValueNanos(10))
             .threadElementsSnapshotCount(1)
             .ignoreIdleThreads(false);
 
-        String memInnerResult = hotThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId);
+        memInnerResult = hotThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId, (interval) -> null);
         assertThat(memInnerResult, containsString("  unique snapshot"));
-        assertThat(memInnerResult, containsString("400b memory allocated by thread 'Thread 4'"));
-        assertThat(memInnerResult, containsString("300b memory allocated by thread 'Thread 3'"));
-        assertThat(memInnerResult, containsString("200b memory allocated by thread 'Thread 2'"));
-        assertThat(memInnerResult, containsString("100b memory allocated by thread 'Thread 1'"));
+        assertThat(memInnerResult, stringContainsInOrder(
+            "400b memory allocated by thread 'Thread 4'",
+            "300b memory allocated by thread 'Thread 3'",
+            "200b memory allocated by thread 'Thread 2'",
+            "100b memory allocated by thread 'Thread 1'"));
+    }
+
+    public void testInnerDetectSingleSnapshot() throws Exception {
+        ThreadMXBean mockedMXBean = makeMockMXBeanHelper();
+        SunThreadInfo mockedSunThreadInfo = makeMockSunThreadInfoHelper();
+
+        long[] threadIds = new long[]{1, 2, 3, 4}; // Adds up to 10, the intervalNanos for calculating time percentages
+        long mockCurrentThreadId = 0L;
+        when(mockedMXBean.getAllThreadIds()).thenReturn(threadIds);
+
+        // Test with only one stack to trigger the different print in innerDetect
+        List<ThreadInfo> allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds);
+        List<ThreadInfo> cpuOrderedInfos = List.of(allInfos.get(3), allInfos.get(2), allInfos.get(1), allInfos.get(0));
+        when(mockedMXBean.getThreadInfo(Matchers.any(), anyInt())).thenReturn(cpuOrderedInfos.toArray(new ThreadInfo[0]));
+
+        HotThreads hotThreads = new HotThreads()
+            .busiestThreads(4)
+            .type(HotThreads.ReportType.CPU)
+            .interval(TimeValue.timeValueNanos(10))
+            .threadElementsSnapshotCount(1)
+            .ignoreIdleThreads(false);
+
+        String singleResult = hotThreads.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId, (interval) -> null);
+
+        assertThat(singleResult, containsString("  unique snapshot"));
+        assertEquals(5, singleResult.split(" unique snapshot").length);
+        assertThat(singleResult, containsString("40.0% [cpu=40.0%, other=0.0%] (4nanos out of 10nanos) cpu usage by thread 'Thread 4'"));
+        assertThat(singleResult, containsString("30.0% [cpu=30.0%, other=0.0%] (3nanos out of 10nanos) cpu usage by thread 'Thread 3'"));
+        assertThat(singleResult, containsString("20.0% [cpu=20.0%, other=0.0%] (2nanos out of 10nanos) cpu usage by thread 'Thread 2'"));
+        assertThat(singleResult, containsString("10.0% [cpu=10.0%, other=0.0%] (1nanos out of 10nanos) cpu usage by thread 'Thread 1'"));
+        assertThat(singleResult, containsString("org.elasticsearch.monitor.test.method_0(Some_File:1)"));
+        assertThat(singleResult, containsString("org.elasticsearch.monitor.test.method_1(Some_File:1)"));
+        assertThat(singleResult, containsString("org.elasticsearch.monitor.testOther.methodFinal(Some_File:1)"));
     }
 
     public void testEnsureInnerDetectSkipsCurrentThread() throws Exception {
-        ThreadMXBean mockedMXBean = mock(ThreadMXBean.class);
-        when(mockedMXBean.isThreadCpuTimeSupported()).thenReturn(true);
+        ThreadMXBean mockedMXBean = makeMockMXBeanHelper();
 
         long mockCurrentThreadId = 5L;
         long[] threadIds = new long[]{mockCurrentThreadId}; // Matches half the intervalNanos for calculating time percentages
@@ -383,7 +543,7 @@ public void testEnsureInnerDetectSkipsCurrentThread() throws Exception {
             .threadElementsSnapshotCount(11)
             .ignoreIdleThreads(false);
 
-        String innerResult = hotThreads.innerDetect(mockedMXBean, mock(SunThreadInfo.class), mockCurrentThreadId);
+        String innerResult = hotThreads.innerDetect(mockedMXBean, mock(SunThreadInfo.class), mockCurrentThreadId, (interval) -> null);
 
         assertEquals(1, innerResult.lines().count());
     }
@@ -394,11 +554,14 @@ public void testReportTypeValueGetter() {
         when(mockedThreadInfo.getBlockedTime()).thenReturn(2L).thenReturn(0L);
         when(mockedThreadInfo.getWaitedTime()).thenReturn(3L).thenReturn(0L);
 
-        HotThreads.ThreadTimeAccumulator info = new HotThreads.ThreadTimeAccumulator(mockedThreadInfo, 1L, 4L);
+        HotThreads.ThreadTimeAccumulator info = new HotThreads.ThreadTimeAccumulator(
+            mockedThreadInfo, new TimeValue(10L), 1L, 4L);
 
         assertEquals(1L, HotThreads.ThreadTimeAccumulator.valueGetterForReportType(HotThreads.ReportType.CPU).applyAsLong(info));
-        assertEquals(3L, HotThreads.ThreadTimeAccumulator.valueGetterForReportType(HotThreads.ReportType.WAIT).applyAsLong(info));
-        assertEquals(2L, HotThreads.ThreadTimeAccumulator.valueGetterForReportType(HotThreads.ReportType.BLOCK).applyAsLong(info));
+        assertEquals(nanosHelper(3L),
+            HotThreads.ThreadTimeAccumulator.valueGetterForReportType(HotThreads.ReportType.WAIT).applyAsLong(info));
+        assertEquals(nanosHelper(2L),
+            HotThreads.ThreadTimeAccumulator.valueGetterForReportType(HotThreads.ReportType.BLOCK).applyAsLong(info));
         assertEquals(4L, HotThreads.ThreadTimeAccumulator.valueGetterForReportType(HotThreads.ReportType.MEM).applyAsLong(info));
 
         //Ensure all enum types have a report type getter
@@ -438,7 +601,7 @@ public void testGetAllValidThreadInfos() {
             assertEquals(0, accumulator.getWaitedTime());
         }
 
-        // Fake sleep, e.g don't sleep call the mock again
+        // Fake sleep, e.g. don't sleep call the mock again
 
         Map<Long, HotThreads.ThreadTimeAccumulator> afterValidInfos = hotThreads.getAllValidThreadInfos(mockedMXBean, mockedSunThreadInfo,
             mockCurrentThreadId);
@@ -449,8 +612,8 @@ public void testGetAllValidThreadInfos() {
             boolean evenThreadId = ((threadId % 2) == 0);
 
             assertEquals(threadId, accumulator.getCpuTime());
-            assertEquals((evenThreadId) ? 0 : threadId, accumulator.getBlockedTime());
-            assertEquals((evenThreadId) ? threadId : 0, accumulator.getWaitedTime());
+            assertEquals(nanosHelper((evenThreadId) ? 0 : threadId), accumulator.getBlockedTime());
+            assertEquals(nanosHelper((evenThreadId) ? threadId : 0), accumulator.getWaitedTime());
         }
 
         // Test when a thread has terminated during sleep, we don't report that thread
@@ -476,8 +639,8 @@ public void testGetAllValidThreadInfos() {
             boolean evenThreadId = ((threadId % 2) == 0);
 
             assertEquals(threadId, accumulator.getCpuTime());
-            assertEquals((evenThreadId) ? 0 : threadId, accumulator.getBlockedTime());
-            assertEquals((evenThreadId) ? threadId : 0, accumulator.getWaitedTime());
+            assertEquals(nanosHelper((evenThreadId) ? 0 : threadId), accumulator.getBlockedTime());
+            assertEquals(nanosHelper((evenThreadId) ? threadId : 0), accumulator.getWaitedTime());
         }
 
         // Test capturing timings for thread that launched while we were sleeping
@@ -493,8 +656,8 @@ public void testGetAllValidThreadInfos() {
 
         HotThreads.ThreadTimeAccumulator firstAccumulator = afterValidInfos.get(removedInfo.getThreadId());
         assertEquals(1, firstAccumulator.getCpuTime());
-        assertEquals(0, firstAccumulator.getWaitedTime());
-        assertEquals(1, firstAccumulator.getBlockedTime());
+        assertEquals(nanosHelper(0), firstAccumulator.getWaitedTime());
+        assertEquals(nanosHelper(1), firstAccumulator.getBlockedTime());
 
         // Test skipping of current thread
         validInfos = hotThreads.getAllValidThreadInfos(mockedMXBean, mockedSunThreadInfo, threadIds[threadIds.length - 1]);
@@ -528,7 +691,7 @@ public void testCaptureThreadStacks() throws InterruptedException {
             .threadElementsSnapshotCount(3)
             .ignoreIdleThreads(false);
 
-        // Setup the mocks
+        // Set up the mocks
         List<ThreadInfo> allInfos = makeThreadInfoMocksHelper(mockedMXBean, threadIds);
 
         long[] topThreadIds = new long[]{threadIds[threadIds.length - 1], threadIds[threadIds.length - 2]};
@@ -555,17 +718,24 @@ public void testThreadInfoAccumulator() {
         ThreadInfo threadOne = makeThreadInfoMocksHelper(mockedMXBean, 1L);
         ThreadInfo threadTwo = makeThreadInfoMocksHelper(mockedMXBean, 2L);
 
-        HotThreads.ThreadTimeAccumulator acc = new HotThreads.ThreadTimeAccumulator(threadOne, 100L, 1000L);
-        HotThreads.ThreadTimeAccumulator accNext = new HotThreads.ThreadTimeAccumulator(threadOne, 250L, 2500L);
+        TimeValue maxTime = new TimeValue(1000L);
+
+        HotThreads.ThreadTimeAccumulator acc = new HotThreads.ThreadTimeAccumulator(
+            threadOne, maxTime, 100L, 1000L);
+        HotThreads.ThreadTimeAccumulator accNext = new HotThreads.ThreadTimeAccumulator(
+            threadOne, maxTime,  250L, 2500L);
         accNext.subtractPrevious(acc);
 
         assertEquals(1500, accNext.getAllocatedBytes());
         assertEquals(150, accNext.getCpuTime());
         assertEquals(0, accNext.getWaitedTime());
-        assertEquals(1, accNext.getBlockedTime());
+        assertEquals(nanosHelper(1), accNext.getBlockedTime());
+        assertEquals(nanosHelper(999), accNext.getRunnableTime());
 
-        HotThreads.ThreadTimeAccumulator accNotMoving = new HotThreads.ThreadTimeAccumulator(threadOne, 250L, 2500L);
-        HotThreads.ThreadTimeAccumulator accNotMovingNext = new HotThreads.ThreadTimeAccumulator(threadOne, 250L, 2500L);
+        HotThreads.ThreadTimeAccumulator accNotMoving = new HotThreads.ThreadTimeAccumulator(
+            threadOne, maxTime, 250L, 2500L);
+        HotThreads.ThreadTimeAccumulator accNotMovingNext = new HotThreads.ThreadTimeAccumulator(
+            threadOne, maxTime, 250L, 2500L);
 
         accNotMovingNext.subtractPrevious(accNotMoving);
 
@@ -573,9 +743,12 @@ public void testThreadInfoAccumulator() {
         assertEquals(0, accNotMovingNext.getCpuTime());
         assertEquals(0, accNotMovingNext.getWaitedTime());
         assertEquals(0, accNotMovingNext.getBlockedTime());
+        assertEquals(0, accNotMovingNext.getRunnableTime());
 
-        HotThreads.ThreadTimeAccumulator accOne = new HotThreads.ThreadTimeAccumulator(threadOne, 250L, 2500L);
-        HotThreads.ThreadTimeAccumulator accTwo = new HotThreads.ThreadTimeAccumulator(threadTwo, 350L, 3500L);
+        HotThreads.ThreadTimeAccumulator accOne = new HotThreads.ThreadTimeAccumulator(
+            threadOne, maxTime, 250L, 2500L);
+        HotThreads.ThreadTimeAccumulator accTwo = new HotThreads.ThreadTimeAccumulator(
+            threadTwo, maxTime, 350L, 3500L);
 
         expectThrows(IllegalStateException.class, () -> accTwo.subtractPrevious(accOne));
     }
@@ -600,51 +773,21 @@ public void testWaitBlockTimeMonitoringEnabled() throws Exception {
             .threadElementsSnapshotCount(11)
             .ignoreIdleThreads(false);
 
-        String innerResult = hotThreads.innerDetect(mockedMXBean, mock(SunThreadInfo.class), mockCurrentThreadId);
-
-        assertThat(innerResult, containsString("Hot threads at "));
-        assertThat(innerResult, containsString("interval=10nanos, busiestThreads=4, ignoreIdleThreads=false:"));
-        assertThat(innerResult, containsString("11/11 snapshots sharing following 2 elements"));
-        assertThat(innerResult, containsString("40.0% (4nanos out of 10nanos) cpu usage by thread 'Thread 4'"));
-        assertThat(innerResult, containsString("30.0% (3nanos out of 10nanos) cpu usage by thread 'Thread 3'"));
-        assertThat(innerResult, containsString("20.0% (2nanos out of 10nanos) cpu usage by thread 'Thread 2'"));
-        assertThat(innerResult, containsString("10.0% (1nanos out of 10nanos) cpu usage by thread 'Thread 1'"));
-        assertThat(innerResult, containsString("org.elasticsearch.monitor.test.method_0(Some_File:1)"));
-        assertThat(innerResult, containsString("org.elasticsearch.monitor.test.method_1(Some_File:1)"));
-        assertThat(innerResult, containsString("org.elasticsearch.monitor.testOther.methodFinal(Some_File:1)"));
-
-        // Ensure we called the monitoring enabled with true and then with false
-        InOrder orderVerifier = inOrder(mockedMXBean);
-        orderVerifier.verify(mockedMXBean).setThreadContentionMonitoringEnabled(true);
-        orderVerifier.verify(mockedMXBean).setThreadContentionMonitoringEnabled(false);
-    }
-
-    public void testWaitBlockTimeMonitoringEnabledWithException() {
-        ThreadMXBean mockedMXBean = mock(ThreadMXBean.class);
-        when(mockedMXBean.isThreadCpuTimeSupported()).thenReturn(true);
-        when(mockedMXBean.isThreadContentionMonitoringSupported()).thenReturn(true);
-
-        when(mockedMXBean.getAllThreadIds()).thenThrow(new RuntimeException("Something bad happened"));
-
-        HotThreads hotThreads = new HotThreads()
-            .busiestThreads(4)
-            .type(HotThreads.ReportType.CPU)
-            .interval(TimeValue.timeValueNanos(10))
-            .threadElementsSnapshotCount(11)
-            .ignoreIdleThreads(false);
-
-        expectThrows(RuntimeException.class, () -> hotThreads.innerDetect(mockedMXBean, mock(SunThreadInfo.class), 0L));
+        Exception e = expectThrows(ElasticsearchException.class, () ->
+            hotThreads.innerDetect(mockedMXBean, mock(SunThreadInfo.class), mockCurrentThreadId, (interval) -> null));
+        assertEquals(e.getMessage(), "thread wait/blocked time accounting is not supported on this JDK");
 
-        // Ensure we called the monitoring enabled with true and then with false even with exception thrown
+        // Ensure we checked if JVM lock monitoring is enabled
         InOrder orderVerifier = inOrder(mockedMXBean);
-        orderVerifier.verify(mockedMXBean).setThreadContentionMonitoringEnabled(true);
-        orderVerifier.verify(mockedMXBean).setThreadContentionMonitoringEnabled(false);
+        orderVerifier.verify(mockedMXBean).isThreadContentionMonitoringSupported();
+        orderVerifier.verify(mockedMXBean).isThreadContentionMonitoringEnabled();
     }
 
     public void testGetThreadAllocatedBytesFailures() throws Exception {
         ThreadMXBean mockedMXBean = mock(ThreadMXBean.class);
         when(mockedMXBean.isThreadCpuTimeSupported()).thenReturn(true);
         when(mockedMXBean.isThreadContentionMonitoringSupported()).thenReturn(true);
+        when(mockedMXBean.isThreadContentionMonitoringEnabled()).thenReturn(true);
 
         SunThreadInfo mockedSunThreadInfo = mock(SunThreadInfo.class);
         when(mockedSunThreadInfo.isThreadAllocatedMemorySupported()).thenReturn(false);
@@ -665,26 +808,7 @@ public void testGetThreadAllocatedBytesFailures() throws Exception {
             .ignoreIdleThreads(false);
 
         ElasticsearchException exception = expectThrows(ElasticsearchException.class,
-            () -> hotThreads0.innerDetect(mockedMXBean, mockedSunThreadInfo, 0L));
+            () -> hotThreads0.innerDetect(mockedMXBean, mockedSunThreadInfo, 0L, (interval) -> null));
         assertThat(exception.getMessage(), equalTo("thread allocated memory is not supported on this JDK"));
-
-        // making sure CPU type was not affected when isThreadAllocatedMemorySupported() == false
-
-        HotThreads hotThreads1 = new HotThreads()
-            .busiestThreads(4)
-            .type(HotThreads.ReportType.CPU)
-            .interval(TimeValue.timeValueNanos(10))
-            .threadElementsSnapshotCount(1)
-            .ignoreIdleThreads(false);
-
-        String innerResult = hotThreads1.innerDetect(mockedMXBean, mockedSunThreadInfo, mockCurrentThreadId);
-        assertThat(innerResult, containsString("Hot threads at "));
-        assertThat(innerResult, containsString("40.0% (4nanos out of 10nanos) cpu usage by thread 'Thread 4'"));
-        assertThat(innerResult, containsString("30.0% (3nanos out of 10nanos) cpu usage by thread 'Thread 3'"));
-        assertThat(innerResult, containsString("20.0% (2nanos out of 10nanos) cpu usage by thread 'Thread 2'"));
-        assertThat(innerResult, containsString("10.0% (1nanos out of 10nanos) cpu usage by thread 'Thread 1'"));
-        assertThat(innerResult, containsString("org.elasticsearch.monitor.test.method_0(Some_File:1)"));
-        assertThat(innerResult, containsString("org.elasticsearch.monitor.test.method_1(Some_File:1)"));
-        assertThat(innerResult, containsString("org.elasticsearch.monitor.testOther.methodFinal(Some_File:1)"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java b/server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java
new file mode 100644
index 0000000000000..542c4c39a2b97
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.test.AbstractDiffableSerializationTestCase;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class FeatureMigrationResultsTests extends AbstractDiffableSerializationTestCase<Metadata.Custom> {
+
+    @Override
+    protected FeatureMigrationResults createTestInstance() {
+        return new FeatureMigrationResults(randomMap(0, 10, () -> new Tuple<>(randomAlphaOfLength(5), randomFeatureStatus())));
+    }
+
+    private SingleFeatureMigrationResult randomFeatureStatus() {
+        if (randomBoolean()) {
+            return SingleFeatureMigrationResult.success();
+        } else {
+            return SingleFeatureMigrationResult.failure(
+                randomAlphaOfLength(8),
+                new RuntimeException(randomAlphaOfLength(5))
+            );
+        }
+    }
+
+    @Override
+    protected FeatureMigrationResults mutateInstance(Metadata.Custom instance) {
+        int oldSize = ((FeatureMigrationResults) instance).getFeatureStatuses().size();
+        if (oldSize == 0 || randomBoolean()) {
+            return new FeatureMigrationResults(
+                randomMap(oldSize + 1, oldSize + 5, () -> new Tuple<>(randomAlphaOfLength(5), randomFeatureStatus()))
+            );
+        } else {
+            return new FeatureMigrationResults(randomMap(0, oldSize, () -> new Tuple<>(randomAlphaOfLength(5), randomFeatureStatus())));
+        }
+    }
+
+    /**
+     * Disable assertions of XContent equivalence - the exception prevents this from working as it translates everything
+     * into ElasticsearchException.
+     */
+    @Override protected boolean assertToXContentEquivalence() {
+        return false;
+    }
+
+    @Override
+    protected Writeable.Reader<Metadata.Custom> instanceReader() {
+        return FeatureMigrationResults::new;
+    }
+
+    @Override protected FeatureMigrationResults doParseInstance(XContentParser parser) throws IOException {
+        return FeatureMigrationResults.fromXContent(parser);
+    }
+
+    @Override protected Metadata.Custom makeTestChanges(Metadata.Custom testInstance) {
+        return mutateInstance(testInstance);
+    }
+
+    @Override protected Writeable.Reader<Diff<Metadata.Custom>> diffReader() {
+        return FeatureMigrationResults.ResultsDiff::new;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java b/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java
new file mode 100644
index 0000000000000..1fa9cb905dc5c
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.AbstractNamedWriteableTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+
+public class SystemIndexMigrationTaskParamsTests extends AbstractNamedWriteableTestCase<SystemIndexMigrationTaskParams> {
+
+    // NOTE: This test case does not currently implement mutateInstance, because all instances of the class
+    // are equal and have the same hashcode (for now).
+
+    @Override
+    protected SystemIndexMigrationTaskParams createTestInstance() {
+        return new SystemIndexMigrationTaskParams();
+    }
+
+    @Override
+    protected SystemIndexMigrationTaskParams copyInstance(SystemIndexMigrationTaskParams instance, Version version) throws IOException {
+        return new SystemIndexMigrationTaskParams();
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(
+            Collections.singletonList(
+                new NamedWriteableRegistry.Entry(
+                    SystemIndexMigrationTaskParams.class,
+                    SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME,
+                    SystemIndexMigrationTaskParams::new
+                )
+            )
+        );
+    }
+
+    @Override
+    protected Class<SystemIndexMigrationTaskParams> categoryClass() {
+        return SystemIndexMigrationTaskParams.class;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java b/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java
new file mode 100644
index 0000000000000..7b8a75ce77588
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.test.AbstractNamedWriteableTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+public class SystemIndexMigrationTaskStateTests extends AbstractNamedWriteableTestCase<SystemIndexMigrationTaskState> {
+
+    @Override
+    protected SystemIndexMigrationTaskState createTestInstance() {
+        return new SystemIndexMigrationTaskState(
+            randomAlphaOfLengthBetween(5, 20),
+            randomAlphaOfLengthBetween(5, 20),
+            randomFeatureCallbackMetadata()
+        );
+    }
+
+    private Map<String, Object> randomFeatureCallbackMetadata() {
+        return randomMap(0, 10, () -> new Tuple<>(randomAlphaOfLength(5), randomMetadataValue()));
+    }
+
+    private Object randomMetadataValue() {
+        switch (randomIntBetween(0, 7)) {
+            case 0:
+                return randomMap(0, 3, () -> new Tuple<>(randomAlphaOfLength(5), randomMetadataValue()));
+            case 1:
+                return randomList(0, 3, this::randomMetadataValue);
+            case 2:
+                return randomLong();
+            case 3:
+                return randomShort();
+            case 4:
+                return randomBoolean();
+            case 5:
+                return randomFloat();
+            case 6:
+                return randomDouble();
+            case 7:
+                return randomAlphaOfLengthBetween(5, 10);
+        }
+        throw new AssertionError("bad randomization");
+    }
+
+    @Override
+    protected SystemIndexMigrationTaskState copyInstance(SystemIndexMigrationTaskState instance, Version version) throws IOException {
+        return new SystemIndexMigrationTaskState(
+            instance.getCurrentIndex(),
+            instance.getCurrentFeature(),
+            instance.getFeatureCallbackMetadata()
+        );
+    }
+
+    @Override protected SystemIndexMigrationTaskState mutateInstance(SystemIndexMigrationTaskState instance) throws IOException {
+        String index = instance.getCurrentIndex();
+        String feature = instance.getCurrentFeature();
+        Map<String, Object> featureMetadata = instance.getFeatureCallbackMetadata();
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                index = randomValueOtherThan(instance.getCurrentIndex(), () -> randomAlphaOfLengthBetween(5, 20));
+                break;
+            case 1:
+                feature = randomValueOtherThan(instance.getCurrentFeature(), () -> randomAlphaOfLengthBetween(5, 20));
+                break;
+            case 2:
+                featureMetadata = randomValueOtherThan(instance.getFeatureCallbackMetadata(), this::randomFeatureCallbackMetadata);
+                break;
+            default:
+                assert false : "invalid randomization case";
+        }
+        return new SystemIndexMigrationTaskState(index, feature, featureMetadata);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(
+            Collections.singletonList(
+                new NamedWriteableRegistry.Entry(
+                    SystemIndexMigrationTaskState.class,
+                    SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME,
+                    SystemIndexMigrationTaskState::new
+                )
+            )
+        );
+    }
+
+    @Override
+    protected Class<SystemIndexMigrationTaskState> categoryClass() {
+        return SystemIndexMigrationTaskState.class;
+    }
+}
diff --git a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh
index 815a9e94e8cb5..da6480d891af7 100644
--- a/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh
+++ b/test/fixtures/krb5kdc-fixture/src/main/resources/provision/peppa.sh
@@ -5,9 +5,9 @@ set -e
 addprinc.sh elasticsearch
 addprinc.sh HTTP/localhost
 addprinc.sh peppa
-addprinc.sh george          dino
+addprinc.sh george          dino_but_longer_than_14_chars
 
 # Use this as a signal that setup is complete
 python3 -m http.server 4444 &
 
-sleep infinity
\ No newline at end of file
+sleep infinity
diff --git a/x-pack/docs/en/security/authentication/built-in-users.asciidoc b/x-pack/docs/en/security/authentication/built-in-users.asciidoc
index a71e3c80f23cd..75ae6addc6e8f 100644
--- a/x-pack/docs/en/security/authentication/built-in-users.asciidoc
+++ b/x-pack/docs/en/security/authentication/built-in-users.asciidoc
@@ -179,7 +179,7 @@ for these users.
 The `apm_system` user is used internally within APM when monitoring is enabled.
 
 To enable this feature in APM, you need to update the
-{apm-server-ref}/configuring-howto-apm-server.html[APM configuration file] to
+`apm-server.yml` configuration file to
 reference the correct username and password. For example:
 
 [source,yaml]
@@ -188,7 +188,6 @@ xpack.monitoring.elasticsearch.username: apm_system
 xpack.monitoring.elasticsearch.password: apmserverpassword
 ----------------------------------------------------------
 
-See {apm-server-ref}/monitoring.html[Monitoring APM Server].
 
 If you have upgraded from an older version of {es}, then you may not have set a
 password for the `apm_system` user. If this is the case,
diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
index 676d4739f02c6..780fb3a6f8b1f 100644
--- a/x-pack/plugin/build.gradle
+++ b/x-pack/plugin/build.gradle
@@ -98,6 +98,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure{ task ->
   task.skipTest("vectors/50_vector_stats/Usage stats on vector fields", "not supported for compatibility")
   task.skipTest("roles/30_prohibited_role_query/Test use prohibited query inside role query", "put role request with a term lookup (deprecated) and type. Requires validation in REST layer")
   task.skipTest("ml/jobs_crud/Test create job with delimited format", "removing undocumented functionality")
+  task.skipTest("ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats", "https://github.com/elastic/elasticsearch/issues/79518")
   task.skipTest("ml/datafeeds_crud/Test update datafeed to point to missing job", "behaviour change #44752 - not allowing to update datafeed job_id")
   task.skipTest("ml/datafeeds_crud/Test update datafeed to point to different job", "behaviour change #44752 - not allowing to update datafeed job_id")
   task.skipTest("ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed", "behaviour change #44752 - not allowing to update datafeed job_id")
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsRequest.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsRequest.java
index 51212d4f9e2d4..cf5466484a099 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsRequest.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsRequest.java
@@ -11,11 +11,13 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.transport.RawIndexingDataTransportRequest;
 
 import java.io.IOException;
 import java.util.List;
 
-public final class BulkShardOperationsRequest extends ReplicatedWriteRequest<BulkShardOperationsRequest> {
+public final class BulkShardOperationsRequest extends ReplicatedWriteRequest<BulkShardOperationsRequest>
+    implements RawIndexingDataTransportRequest {
 
     private final String historyUUID;
     private final List<Translog.Operation> operations;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
index 0aa300346c58d..8d250900a11b3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
@@ -60,6 +60,7 @@ public static class Request extends MasterNodeRequest<Request> implements ToXCon
         public static final ParseField WAIT_FOR = new ParseField("wait_for");
         public static final ParseField INFERENCE_THREADS = TaskParams.INFERENCE_THREADS;
         public static final ParseField MODEL_THREADS = TaskParams.MODEL_THREADS;
+        public static final ParseField QUEUE_CAPACITY = TaskParams.QUEUE_CAPACITY;
 
         public static final ObjectParser<Request, Void> PARSER = new ObjectParser<>(NAME, Request::new);
 
@@ -69,6 +70,7 @@ public static class Request extends MasterNodeRequest<Request> implements ToXCon
             PARSER.declareString((request, waitFor) -> request.setWaitForState(AllocationStatus.State.fromString(waitFor)), WAIT_FOR);
             PARSER.declareInt(Request::setInferenceThreads, INFERENCE_THREADS);
             PARSER.declareInt(Request::setModelThreads, MODEL_THREADS);
+            PARSER.declareInt(Request::setQueueCapacity, QUEUE_CAPACITY);
         }
 
         public static Request parseRequest(String modelId, XContentParser parser) {
@@ -87,6 +89,7 @@ public static Request parseRequest(String modelId, XContentParser parser) {
         private AllocationStatus.State waitForState = AllocationStatus.State.STARTED;
         private int modelThreads = 1;
         private int inferenceThreads = 1;
+        private int queueCapacity = 1024;
 
         private Request() {}
 
@@ -101,6 +104,7 @@ public Request(StreamInput in) throws IOException {
             waitForState = in.readEnum(AllocationStatus.State.class);
             modelThreads = in.readVInt();
             inferenceThreads = in.readVInt();
+            queueCapacity = in.readVInt();
         }
 
         public final void setModelId(String modelId) {
@@ -144,6 +148,14 @@ public void setInferenceThreads(int inferenceThreads) {
             this.inferenceThreads = inferenceThreads;
         }
 
+        public int getQueueCapacity() {
+            return queueCapacity;
+        }
+
+        public void setQueueCapacity(int queueCapacity) {
+            this.queueCapacity = queueCapacity;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
@@ -152,6 +164,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeEnum(waitForState);
             out.writeVInt(modelThreads);
             out.writeVInt(inferenceThreads);
+            out.writeVInt(queueCapacity);
         }
 
         @Override
@@ -162,6 +175,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(WAIT_FOR.getPreferredName(), waitForState);
             builder.field(MODEL_THREADS.getPreferredName(), modelThreads);
             builder.field(INFERENCE_THREADS.getPreferredName(), inferenceThreads);
+            builder.field(QUEUE_CAPACITY.getPreferredName(), queueCapacity);
             builder.endObject();
             return builder;
         }
@@ -183,12 +197,15 @@ public ActionRequestValidationException validate() {
             if (inferenceThreads < 1) {
                 validationException.addValidationError("[" + INFERENCE_THREADS + "] must be a positive integer");
             }
+            if (queueCapacity < 1) {
+                validationException.addValidationError("[" + QUEUE_CAPACITY + "] must be a positive integer");
+            }
             return validationException.validationErrors().isEmpty() ? null : validationException;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(modelId, timeout, waitForState, modelThreads, inferenceThreads);
+            return Objects.hash(modelId, timeout, waitForState, modelThreads, inferenceThreads, queueCapacity);
         }
 
         @Override
@@ -204,7 +221,8 @@ public boolean equals(Object obj) {
                 && Objects.equals(timeout, other.timeout)
                 && Objects.equals(waitForState, other.waitForState)
                 && modelThreads == other.modelThreads
-                && inferenceThreads == other.inferenceThreads;
+                && inferenceThreads == other.inferenceThreads
+                && queueCapacity == other.queueCapacity;
         }
 
         @Override
@@ -226,16 +244,20 @@ public static boolean mayAllocateToNode(DiscoveryNode node) {
         private static final ParseField MODEL_BYTES = new ParseField("model_bytes");
         public static final ParseField MODEL_THREADS = new ParseField("model_threads");
         public static final ParseField INFERENCE_THREADS = new ParseField("inference_threads");
+        public static final ParseField QUEUE_CAPACITY = new ParseField("queue_capacity");
+
         private static final ConstructingObjectParser<TaskParams, Void> PARSER = new ConstructingObjectParser<>(
             "trained_model_deployment_params",
             true,
-            a -> new TaskParams((String)a[0], (Long)a[1], (int) a[2], (int) a[3])
+            a -> new TaskParams((String)a[0], (Long)a[1], (int) a[2], (int) a[3], (int) a[4])
         );
+
         static {
             PARSER.declareString(ConstructingObjectParser.constructorArg(), TrainedModelConfig.MODEL_ID);
             PARSER.declareLong(ConstructingObjectParser.constructorArg(), MODEL_BYTES);
             PARSER.declareInt(ConstructingObjectParser.constructorArg(), INFERENCE_THREADS);
             PARSER.declareInt(ConstructingObjectParser.constructorArg(), MODEL_THREADS);
+            PARSER.declareInt(ConstructingObjectParser.constructorArg(), QUEUE_CAPACITY);
         }
 
         public static TaskParams fromXContent(XContentParser parser) {
@@ -253,28 +275,22 @@ public static TaskParams fromXContent(XContentParser parser) {
         private final long modelBytes;
         private final int inferenceThreads;
         private final int modelThreads;
+        private final int queueCapacity;
 
-        public TaskParams(String modelId, long modelBytes, int inferenceThreads, int modelThreads) {
+        public TaskParams(String modelId, long modelBytes, int inferenceThreads, int modelThreads, int queueCapacity) {
             this.modelId = Objects.requireNonNull(modelId);
             this.modelBytes = modelBytes;
-            if (modelBytes < 0) {
-                throw new IllegalArgumentException("modelBytes must be non-negative");
-            }
             this.inferenceThreads = inferenceThreads;
-            if (inferenceThreads < 1) {
-                throw new IllegalArgumentException(INFERENCE_THREADS + " must be positive");
-            }
             this.modelThreads = modelThreads;
-            if (modelThreads < 1) {
-                throw new IllegalArgumentException(MODEL_THREADS + " must be positive");
-            }
+            this.queueCapacity = queueCapacity;
         }
 
         public TaskParams(StreamInput in) throws IOException {
             this.modelId = in.readString();
-            this.modelBytes = in.readVLong();
+            this.modelBytes = in.readLong();
             this.inferenceThreads = in.readVInt();
             this.modelThreads = in.readVInt();
+            this.queueCapacity = in.readVInt();
         }
 
         public String getModelId() {
@@ -293,9 +309,10 @@ public Version getMinimalSupportedVersion() {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(modelId);
-            out.writeVLong(modelBytes);
+            out.writeLong(modelBytes);
             out.writeVInt(inferenceThreads);
             out.writeVInt(modelThreads);
+            out.writeVInt(queueCapacity);
         }
 
         @Override
@@ -305,13 +322,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(MODEL_BYTES.getPreferredName(), modelBytes);
             builder.field(INFERENCE_THREADS.getPreferredName(), inferenceThreads);
             builder.field(MODEL_THREADS.getPreferredName(), modelThreads);
+            builder.field(QUEUE_CAPACITY.getPreferredName(), queueCapacity);
             builder.endObject();
             return builder;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(modelId, modelBytes, inferenceThreads, modelThreads);
+            return Objects.hash(modelId, modelBytes, inferenceThreads, modelThreads, queueCapacity);
         }
 
         @Override
@@ -323,7 +341,8 @@ public boolean equals(Object o) {
             return Objects.equals(modelId, other.modelId)
                 && modelBytes == other.modelBytes
                 && inferenceThreads == other.inferenceThreads
-                && modelThreads == other.modelThreads;
+                && modelThreads == other.modelThreads
+                && queueCapacity == other.queueCapacity;
         }
 
         @Override
@@ -342,6 +361,15 @@ public int getInferenceThreads() {
         public int getModelThreads() {
             return modelThreads;
         }
+
+        public int getQueueCapacity() {
+            return queueCapacity;
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
     }
 
     public interface TaskMatcher {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
index 80018208e8c6d..9d2b931013952 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -25,11 +26,13 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
 
+import java.util.List;
 import java.util.SortedMap;
 
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
@@ -41,8 +44,15 @@ public class AnnotationIndex {
 
     public static final String READ_ALIAS_NAME = ".ml-annotations-read";
     public static final String WRITE_ALIAS_NAME = ".ml-annotations-write";
-    // Exposed for testing, but always use the aliases in non-test code
-    public static final String INDEX_NAME = ".ml-annotations-6";
+
+    // Exposed for testing, but always use the aliases in non-test code.
+    public static final String LATEST_INDEX_NAME = ".ml-annotations-000001";
+    // Due to historical bugs this index may not have the correct mappings
+    // in some production clusters. Therefore new annotations should be
+    // written to the latest index. If we ever switch to another new annotations
+    // index then this list should be adjusted to include the previous latest
+    // index.
+    public static final List<String> OLD_INDEX_NAMES = List.of(".ml-annotations-6");
 
     private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
 
@@ -85,12 +95,18 @@ public static void createAnnotationsIndexIfNecessary(Client client, ClusterState
             finalListener::onFailure);
 
         final ActionListener<Boolean> createAliasListener = ActionListener.wrap(success -> {
-            final IndicesAliasesRequest request =
+            final IndicesAliasesRequestBuilder requestBuilder =
                 client.admin().indices().prepareAliases()
-                    .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(READ_ALIAS_NAME).isHidden(true))
-                    .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(WRITE_ALIAS_NAME).isHidden(true))
-                    .request();
-            executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
+                    .addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                        .index(LATEST_INDEX_NAME).alias(READ_ALIAS_NAME).isHidden(true))
+                    .addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                        .index(LATEST_INDEX_NAME).alias(WRITE_ALIAS_NAME).isHidden(true));
+            for (String oldIndexName : OLD_INDEX_NAMES) {
+                if (state.getMetadata().getIndicesLookup().containsKey(oldIndexName)) {
+                    requestBuilder.removeAlias(oldIndexName, WRITE_ALIAS_NAME);
+                }
+            }
+            executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, requestBuilder.request(),
                 ActionListener.<AcknowledgedResponse>wrap(
                     r -> checkMappingsListener.onResponse(r.isAcknowledged()), finalListener::onFailure),
                 client.admin().indices()::aliases);
@@ -104,18 +120,18 @@ public static void createAnnotationsIndexIfNecessary(Client client, ClusterState
             mlLookup.isEmpty() == false && mlLookup.firstKey().startsWith(".ml")) {
 
             // Create the annotations index if it doesn't exist already.
-            if (mlLookup.containsKey(INDEX_NAME) == false) {
+            if (mlLookup.containsKey(LATEST_INDEX_NAME) == false) {
                 logger.debug(
                     () -> new ParameterizedMessage(
                         "Creating [{}] because [{}] exists; trace {}",
-                        INDEX_NAME,
+                        LATEST_INDEX_NAME,
                         mlLookup.firstKey(),
                         org.elasticsearch.ExceptionsHelper.formatStackTrace(Thread.currentThread().getStackTrace())
                     )
                 );
 
                 CreateIndexRequest createIndexRequest =
-                    new CreateIndexRequest(INDEX_NAME)
+                    new CreateIndexRequest(LATEST_INDEX_NAME)
                         .mapping(annotationsMapping())
                         .settings(Settings.builder()
                             .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
@@ -140,7 +156,14 @@ public static void createAnnotationsIndexIfNecessary(Client client, ClusterState
             }
 
             // Recreate the aliases if they've gone even though the index still exists.
-            if (mlLookup.containsKey(READ_ALIAS_NAME) == false || mlLookup.containsKey(WRITE_ALIAS_NAME) == false) {
+            IndexAbstraction writeAliasDefinition = mlLookup.get(WRITE_ALIAS_NAME);
+            if (mlLookup.containsKey(READ_ALIAS_NAME) == false || writeAliasDefinition == null) {
+                createAliasListener.onResponse(true);
+                return;
+            }
+
+            List<Index> writeAliasIndices = writeAliasDefinition.getIndices();
+            if (writeAliasIndices.size() != 1 || LATEST_INDEX_NAME.equals(writeAliasIndices.get(0).getName()) == false) {
                 createAliasListener.onResponse(true);
                 return;
             }
@@ -154,7 +177,7 @@ public static void createAnnotationsIndexIfNecessary(Client client, ClusterState
         finalListener.onResponse(false);
     }
 
-    private static String annotationsMapping() {
+    public static String annotationsMapping() {
         return TemplateUtils.loadTemplate(
             "/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE);
     }
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringDeprecatedSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoringDeprecatedSettings.java
similarity index 97%
rename from x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringDeprecatedSettings.java
rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoringDeprecatedSettings.java
index 3b6a5370a5151..1a467a404cf01 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringDeprecatedSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoringDeprecatedSettings.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.monitoring;
+package org.elasticsearch.xpack.core.monitoring;
 
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java
index d97cbaa5b5124..acc387d42bf97 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java
@@ -36,7 +36,11 @@ public final class SystemPrivilege extends Privilege {
         RetentionLeaseActions.Remove.ACTION_NAME + "*", // needed for CCR to remove retention leases
         RetentionLeaseActions.Renew.ACTION_NAME + "*", // needed for CCR to renew retention leases
         "indices:admin/settings/update", // needed for DiskThresholdMonitor.markIndicesReadOnly
-        CompletionPersistentTaskAction.NAME // needed for ShardFollowTaskCleaner
+        CompletionPersistentTaskAction.NAME, // needed for ShardFollowTaskCleaner
+        "indices:data/write/*", // needed for SystemIndexMigrator
+        "indices:data/read/*", // needed for SystemIndexMigrator
+        "indices:admin/refresh", // needed for SystemIndexMigrator
+        "indices:admin/aliases" // needed for SystemIndexMigrator
     );
 
     private static final Predicate<String> PREDICATE = (action) -> {
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json
index 17ebf089de809..1292ce06e3044 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json
@@ -3,6 +3,7 @@
     "_meta" : {
       "version" : "${xpack.ml.version}"
     },
+    "dynamic" : false,
     "properties" : {
       "annotation" : {
         "type" : "text"
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAllocationActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAllocationActionRequestTests.java
index 6a6fa0453ff7e..978139c44e142 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAllocationActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAllocationActionRequestTests.java
@@ -14,14 +14,7 @@ public class CreateTrainedModelAllocationActionRequestTests extends AbstractWire
 
     @Override
     protected Request createTestInstance() {
-        return new Request(
-            new StartTrainedModelDeploymentAction.TaskParams(
-                randomAlphaOfLength(10),
-                randomNonNegativeLong(),
-                randomIntBetween(1, 8),
-                randomIntBetween(1, 8)
-            )
-        );
+        return new Request(StartTrainedModelDeploymentTaskParamsTests.createRandom());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java
index 6bd27634dcf69..4c17f35025e7a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java
@@ -18,6 +18,7 @@
 import java.io.IOException;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
@@ -53,6 +54,9 @@ public static Request createRandom() {
         if (randomBoolean()) {
             request.setModelThreads(randomIntBetween(1, 8));
         }
+        if (randomBoolean()) {
+            request.setQueueCapacity(randomIntBetween(1, 10000));
+        }
         return request;
     }
 
@@ -95,4 +99,33 @@ public void testValidate_GivenModelThreadsIsNegative() {
         assertThat(e, is(not(nullValue())));
         assertThat(e.getMessage(), containsString("[model_threads] must be a positive integer"));
     }
+
+    public void testValidate_GivenQueueCapacityIsZero() {
+        Request request = createRandom();
+        request.setQueueCapacity(0);
+
+        ActionRequestValidationException e = request.validate();
+
+        assertThat(e, is(not(nullValue())));
+        assertThat(e.getMessage(), containsString("[queue_capacity] must be a positive integer"));
+    }
+
+    public void testValidate_GivenQueueCapacityIsNegative() {
+        Request request = createRandom();
+        request.setQueueCapacity(randomIntBetween(Integer.MIN_VALUE, -1));
+
+        ActionRequestValidationException e = request.validate();
+
+        assertThat(e, is(not(nullValue())));
+        assertThat(e.getMessage(), containsString("[queue_capacity] must be a positive integer"));
+    }
+
+    public void testDefaults() {
+        Request request = new Request(randomAlphaOfLength(10));
+        assertThat(request.getTimeout(), equalTo(TimeValue.timeValueSeconds(20)));
+        assertThat(request.getWaitForState(), equalTo(AllocationStatus.State.STARTED));
+        assertThat(request.getInferenceThreads(), equalTo(1));
+        assertThat(request.getModelThreads(), equalTo(1));
+        assertThat(request.getQueueCapacity(), equalTo(1024));
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java
index 95a529d3ccc1e..c5160f96663a3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java
@@ -36,7 +36,8 @@ public static StartTrainedModelDeploymentAction.TaskParams createRandom() {
             randomAlphaOfLength(10),
             randomNonNegativeLong(),
             randomIntBetween(1, 8),
-            randomIntBetween(1, 8)
+            randomIntBetween(1, 8),
+            randomIntBetween(1, 10000)
         );
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/allocation/TrainedModelAllocationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/allocation/TrainedModelAllocationTests.java
index 473730901cac7..82ca307f0e024 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/allocation/TrainedModelAllocationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/allocation/TrainedModelAllocationTests.java
@@ -13,9 +13,10 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
+import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentTaskParamsTests;
 
 import java.io.IOException;
 import java.util.List;
@@ -31,9 +32,7 @@
 public class TrainedModelAllocationTests extends AbstractSerializingTestCase<TrainedModelAllocation> {
 
     public static TrainedModelAllocation randomInstance() {
-        TrainedModelAllocation.Builder builder = TrainedModelAllocation.Builder.empty(
-            new StartTrainedModelDeploymentAction.TaskParams(randomAlphaOfLength(10), randomNonNegativeLong(), 1, 1)
-        );
+        TrainedModelAllocation.Builder builder = TrainedModelAllocation.Builder.empty(randomParams());
         List<String> nodes = Stream.generate(() -> randomAlphaOfLength(10)).limit(randomInt(5)).collect(Collectors.toList());
         for (String node : nodes) {
             if (randomBoolean()) {
@@ -249,7 +248,7 @@ private static DiscoveryNode buildNode() {
     }
 
     private static StartTrainedModelDeploymentAction.TaskParams randomParams() {
-        return new StartTrainedModelDeploymentAction.TaskParams(randomAlphaOfLength(10), randomNonNegativeLong(), 1, 1);
+        return StartTrainedModelDeploymentTaskParamsTests.createRandom();
     }
 
     private static void assertUnchanged(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/PrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/PrivilegeTests.java
index 354bcc97d6fa6..b79dbb5a4a1c2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/PrivilegeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/PrivilegeTests.java
@@ -10,11 +10,11 @@
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.enrich.action.DeleteEnrichPolicyAction;
 import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction;
 import org.elasticsearch.xpack.core.enrich.action.GetEnrichPolicyAction;
 import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction;
-import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authz.permission.ClusterPermission;
 import org.elasticsearch.xpack.core.security.support.Automatons;
@@ -155,7 +155,7 @@ public void testSystem() throws Exception {
         assertThat(predicate.test("cluster:admin/whatever"), is(false));
         assertThat(predicate.test("indices:admin/mapping/put"), is(true));
         assertThat(predicate.test("indices:admin/mapping/whatever"), is(false));
-        assertThat(predicate.test("internal:transport/proxy/indices:data/read/query"), is(false));
+        assertThat(predicate.test("internal:transport/proxy/indices:data/read/query"), is(true));
         assertThat(predicate.test("internal:transport/proxy/indices:monitor/whatever"), is(true));
         assertThat(predicate.test("indices:admin/seq_no/global_checkpoint_sync"), is(true));
         assertThat(predicate.test("indices:admin/seq_no/global_checkpoint_sync[p]"), is(true));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
index ea57084a76b74..7c6cb6d1db757 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
@@ -1478,7 +1478,7 @@ public void testMachineLearningAdminRole() {
         assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX);
         assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT);
         assertOnlyReadAllowed(role, NotificationsIndex.NOTIFICATIONS_INDEX);
-        assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.INDEX_NAME);
+        assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.LATEST_INDEX_NAME);
 
         assertNoAccessAllowed(role, RestrictedIndicesNames.RESTRICTED_NAMES);
         assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2));
@@ -1632,7 +1632,7 @@ public void testMachineLearningUserRole() {
         assertNoAccessAllowed(role, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX);
         assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT);
         assertOnlyReadAllowed(role, NotificationsIndex.NOTIFICATIONS_INDEX);
-        assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.INDEX_NAME);
+        assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.LATEST_INDEX_NAME);
 
         assertNoAccessAllowed(role, RestrictedIndicesNames.RESTRICTED_NAMES);
         assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2));
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java
index f3e5ca87eb8b8..63e3ee50187a7 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java
@@ -14,9 +14,9 @@ public class ClusterDeprecationChecks {
     static DeprecationIssue checkTransientSettingsExistence(ClusterState state) {
         if (state.metadata().transientSettings().isEmpty() == false) {
             return new DeprecationIssue(DeprecationIssue.Level.WARNING,
-                "Transient cluster settings are in the process of being removed.",
+                "Transient cluster settings are deprecated",
                 "https://ela.st/es-deprecation-7-transient-cluster-settings",
-                "Use persistent settings to define your cluster settings instead.",
+                "Use persistent settings to configure your cluster.",
                 false,
                 null);
         }
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
index e850929a5b97d..2593f736bf39f 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
@@ -45,7 +45,10 @@ private DeprecationChecks() {
     static List<BiFunction<Settings, PluginsAndModules, DeprecationIssue>> NODE_SETTINGS_CHECKS = List.of(
         NodeDeprecationChecks::checkSharedDataPathSetting,
         NodeDeprecationChecks::checkReservedPrefixedRealmNames,
-        NodeDeprecationChecks::checkSingleDataNodeWatermarkSetting
+        NodeDeprecationChecks::checkSingleDataNodeWatermarkSetting,
+        NodeDeprecationChecks::checkExporterUseIngestPipelineSettings,
+        NodeDeprecationChecks::checkExporterPipelineMasterTimeoutSetting,
+        NodeDeprecationChecks::checkExporterCreateLegacyTemplateSetting
     );
 
     static List<Function<IndexMetadata, DeprecationIssue>> INDEX_SETTINGS_CHECKS = List.of(
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
index 6d06a61908154..d9fede7274447 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
@@ -13,10 +13,12 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
+import org.elasticsearch.xpack.core.monitoring.MonitoringDeprecatedSettings;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -93,4 +95,48 @@ static DeprecationIssue checkSingleDataNodeWatermarkSetting(final Settings setti
 
         return null;
     }
+
+    private static DeprecationIssue deprecatedAffixSetting(Setting.AffixSetting<?> deprecatedAffixSetting, String detailPattern,
+                                                           String url, DeprecationIssue.Level warningLevel, Settings settings) {
+        List<Setting<?>> deprecatedConcreteSettings = deprecatedAffixSetting.getAllConcreteSettings(settings)
+            .sorted(Comparator.comparing(Setting::getKey)).collect(Collectors.toList());
+
+        if (deprecatedConcreteSettings.isEmpty()) {
+            return null;
+        }
+
+        final String concatSettingNames = deprecatedConcreteSettings.stream().map(Setting::getKey).collect(Collectors.joining(","));
+        final String message = String.format(
+            Locale.ROOT,
+            "The [%s] settings are deprecated and will be removed after 8.0",
+            concatSettingNames
+        );
+        final String details = String.format(Locale.ROOT, detailPattern, concatSettingNames);
+
+        return new DeprecationIssue(warningLevel, message, url, details, false, null);
+    }
+
+    static DeprecationIssue checkExporterUseIngestPipelineSettings(final Settings settings, final PluginsAndModules pluginsAndModules) {
+        return deprecatedAffixSetting(MonitoringDeprecatedSettings.USE_INGEST_PIPELINE_SETTING,
+            "Remove the following settings from elasticsearch.yml: [%s]",
+            "https://ela.st/es-deprecation-7-monitoring-exporter-use-ingest-setting",
+            DeprecationIssue.Level.WARNING,
+            settings);
+    }
+
+    static DeprecationIssue checkExporterPipelineMasterTimeoutSetting(final Settings settings, final PluginsAndModules pluginsAndModules) {
+        return deprecatedAffixSetting(MonitoringDeprecatedSettings.PIPELINE_CHECK_TIMEOUT_SETTING,
+            "Remove the following settings from elasticsearch.yml: [%s]",
+            "https://ela.st/es-deprecation-7-monitoring-exporter-pipeline-timeout-setting",
+            DeprecationIssue.Level.WARNING,
+            settings);
+    }
+
+    static DeprecationIssue checkExporterCreateLegacyTemplateSetting(final Settings settings, final PluginsAndModules pluginsAndModules) {
+        return deprecatedAffixSetting(MonitoringDeprecatedSettings.TEMPLATE_CREATE_LEGACY_VERSIONS_SETTING,
+            "Remove the following settings from elasticsearch.yml: [%s]",
+            "https://ela.st/es-deprecation-7-monitoring-exporter-create-legacy-template-setting",
+            DeprecationIssue.Level.WARNING,
+            settings);
+    }
 }
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java
index abacd55a3acee..730bf691ddc10 100644
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java
@@ -18,10 +18,17 @@
 public class ClusterDeprecationChecksTests extends ESTestCase {
 
     public void testCheckTransientSettingsExistence() {
+        Settings persistentSettings = Settings.builder()
+            .put("xpack.monitoring.collection.enabled", true)
+            .build();
+
         Settings transientSettings = Settings.builder()
             .put("indices.recovery.max_bytes_per_sec", "20mb")
+            .put("action.auto_create_index", true)
+            .put("cluster.routing.allocation.enable", "primaries")
             .build();
         Metadata metadataWithTransientSettings = Metadata.builder()
+            .persistentSettings(persistentSettings)
             .transientSettings(transientSettings)
             .build();
 
@@ -29,13 +36,13 @@ public void testCheckTransientSettingsExistence() {
         DeprecationIssue issue = ClusterDeprecationChecks.checkTransientSettingsExistence(badState);
         assertThat(issue, equalTo(
             new DeprecationIssue(DeprecationIssue.Level.WARNING,
-                "Transient cluster settings are in the process of being removed.",
+                "Transient cluster settings are deprecated",
                 "https://ela.st/es-deprecation-7-transient-cluster-settings",
-                "Use persistent settings to define your cluster settings instead.",
+                "Use persistent settings to configure your cluster.",
                 false, null)
         ));
 
-        Settings persistentSettings = Settings.builder()
+        persistentSettings = Settings.builder()
             .put("indices.recovery.max_bytes_per_sec", "20mb")
             .build();
         Metadata metadataWithoutTransientSettings = Metadata.builder()
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
index 83f2689abf177..42088a0ba89c2 100644
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
@@ -138,4 +139,57 @@ public void testSingleDataNodeWatermarkSetting() {
                     " Discontinue use of this setting.",
                 false, null)));
     }
+
+    public void testExporterUseIngestPipelineSettings() {
+        Settings settings = Settings.builder()
+            .put("xpack.monitoring.exporters.test.use_ingest", true)
+            .build();
+
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, c -> c.apply(settings, null));
+
+        final String expectedUrl =
+            "https://ela.st/es-deprecation-7-monitoring-exporter-use-ingest-setting";
+        assertThat(issues, hasItem(
+            new DeprecationIssue(DeprecationIssue.Level.WARNING,
+                "The [xpack.monitoring.exporters.test.use_ingest] settings are deprecated and will be removed after 8.0",
+                expectedUrl,
+                "Remove the following settings from elasticsearch.yml: [xpack.monitoring.exporters.test.use_ingest]",
+                false, null)));
+    }
+
+    public void testExporterPipelineMasterTimeoutSetting() {
+        Settings settings = Settings.builder()
+            .put("xpack.monitoring.exporters.test.index.pipeline.master_timeout", TimeValue.timeValueSeconds(10))
+            .build();
+
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, c -> c.apply(settings, null));
+
+        final String expectedUrl =
+            "https://ela.st/es-deprecation-7-monitoring-exporter-pipeline-timeout-setting";
+        assertThat(issues, hasItem(
+            new DeprecationIssue(DeprecationIssue.Level.WARNING,
+                "The [xpack.monitoring.exporters.test.index.pipeline.master_timeout] settings are deprecated and will be removed after 8.0",
+                expectedUrl,
+                "Remove the following settings from elasticsearch.yml: [xpack.monitoring.exporters.test.index.pipeline.master_timeout]",
+                false, null)));
+    }
+
+    public void testExporterCreateLegacyTemplateSetting() {
+        Settings settings = Settings.builder()
+            .put("xpack.monitoring.exporters.test.index.template.create_legacy_templates", true)
+            .build();
+
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, c -> c.apply(settings, null));
+
+        final String expectedUrl =
+            "https://ela.st/es-deprecation-7-monitoring-exporter-create-legacy-template-setting";
+        assertThat(issues, hasItem(
+            new DeprecationIssue(DeprecationIssue.Level.WARNING,
+                "The [xpack.monitoring.exporters.test.index.template.create_legacy_templates] settings are deprecated and will be " +
+                    "removed after 8.0",
+                expectedUrl,
+                "Remove the following settings from elasticsearch.yml: " +
+                    "[xpack.monitoring.exporters.test.index.template.create_legacy_templates]",
+                false, null)));
+    }
 }
diff --git a/x-pack/plugin/mapper-unsigned-long/build.gradle b/x-pack/plugin/mapper-unsigned-long/build.gradle
index 4eac800de79d4..00cf286797afa 100644
--- a/x-pack/plugin/mapper-unsigned-long/build.gradle
+++ b/x-pack/plugin/mapper-unsigned-long/build.gradle
@@ -30,4 +30,10 @@ restResources {
     restApi {
         include '_common', 'bulk', 'indices', 'index', 'search', 'xpack'
     }
-}
\ No newline at end of file
+}
+
+tasks.named("yamlRestTestV7CompatTest").configure {
+    systemProperty 'tests.rest.blacklist', [
+        '50_script_values/Scripted fields values return Long'
+    ].join(',')
+}
diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java
index 9739b63e9da7d..a5407719d5f93 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java
+++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java
@@ -20,7 +20,7 @@
 import static org.elasticsearch.search.DocValueFormat.MASK_2_63;
 import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.BIGINTEGER_2_64_MINUS_ONE;
 
-public class UnsignedLongDocValuesField implements UnsignedLongField, DocValuesField<Long> {
+public class UnsignedLongDocValuesField implements UnsignedLongField, DocValuesField {
 
     private final SortedNumericDocValues input;
     private long[] values = new long[0];
@@ -89,26 +89,12 @@ public List<Long> getValues() {
     }
 
     @Override
-    public Long getValue(Long defaultValue) {
+    public long getValue(long defaultValue) {
         return getValue(0, defaultValue);
     }
 
     @Override
-    public Long getValue(int index, Long defaultValue) {
-        if (isEmpty() || index < 0 || index >= count) {
-            return defaultValue;
-        }
-
-        return toFormatted(index);
-    }
-
-    @Override
-    public long getLong(long defaultValue) {
-        return getLong(0, defaultValue);
-    }
-
-    @Override
-    public long getLong(int index, long defaultValue) {
+    public long getValue(int index, long defaultValue) {
         if (isEmpty() || index < 0 || index >= count) {
             return defaultValue;
         }
diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java
index 96b8c730eed69..acf69cd7ae3bd 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java
+++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java
@@ -12,13 +12,16 @@
 import java.math.BigInteger;
 import java.util.List;
 
-public interface UnsignedLongField extends Field<Long> {
+public interface UnsignedLongField extends Field {
 
     /** Returns the 0th index value as an {@code long} if it exists, otherwise {@code defaultValue}. */
-    long getLong(long defaultValue);
+    long getValue(long defaultValue);
 
     /** Returns the value at {@code index} as an {@code long} if it exists, otherwise {@code defaultValue}. */
-    long getLong(int index, long defaultValue);
+    long getValue(int index, long defaultValue);
+
+    /** Return all the values as a {@code List}. */
+    List<Long> getValues();
 
     /** Returns the 0th index value as a {@code BigInteger} if it exists, otherwise {@code defaultValue}. */
     BigInteger getBigInteger(BigInteger defaultValue);
diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java
index 30575818d169a..63e4e6d8dc142 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java
+++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java
@@ -79,7 +79,7 @@ public ScriptDocValues<?> getScriptValues() {
     }
 
     @Override
-    public DocValuesField<?> getScriptField(String name) {
+    public DocValuesField getScriptField(String name) {
         return new UnsignedLongDocValuesField(getLongValues(), name);
     }
 
diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/org.elasticsearch.xpack.unsignedlong.txt b/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/org.elasticsearch.xpack.unsignedlong.txt
index bebec2e2a47da..574a535714d3d 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/org.elasticsearch.xpack.unsignedlong.txt
+++ b/x-pack/plugin/mapper-unsigned-long/src/main/resources/org/elasticsearch/xpack/unsignedlong/org.elasticsearch.xpack.unsignedlong.txt
@@ -12,8 +12,9 @@ class org.elasticsearch.xpack.unsignedlong.UnsignedLongScriptDocValues {
 }
 
 class org.elasticsearch.xpack.unsignedlong.UnsignedLongField @dynamic_type {
-  long getLong(long)
-  long getLong(int, long)
+  long getValue(long)
+  long getValue(int, long)
+  List getValues()
   BigInteger getBigInteger(BigInteger)
   BigInteger getBigInteger(int, BigInteger)
   List getBigIntegers()
diff --git a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/50_script_values.yml b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/50_script_values.yml
index c43f22a193f38..7a4be63265679 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/50_script_values.yml
+++ b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/50_script_values.yml
@@ -46,21 +46,7 @@ setup:
   - match: { hits.hits.2.fields.scripted_ul.0: -9223372036854775808 }
   - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 }
   - match: { hits.hits.4.fields.scripted_ul.0: 0 }
-  - do:
-      search:
-        index: test1
-        body:
-          sort: [ { ul: desc } ]
-          script_fields:
-            scripted_ul:
-              script:
-                source: "field('ul').getLong(1000L)"
 
-  - match: { hits.hits.0.fields.scripted_ul.0: -1 }
-  - match: { hits.hits.1.fields.scripted_ul.0: -3 }
-  - match: { hits.hits.2.fields.scripted_ul.0: -9223372036854775808 }
-  - match: { hits.hits.3.fields.scripted_ul.0: 9223372036854775807 }
-  - match: { hits.hits.4.fields.scripted_ul.0: 0 }
   - do:
       search:
         index: test1
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
index 73bab18e0458d..88f836d10f730 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java
@@ -264,7 +264,7 @@ protected void waitForecastStatus(int maxWaitTimeSeconds,
 
     protected void assertThatNumberOfAnnotationsIsEqualTo(int expectedNumberOfAnnotations) throws IOException {
         // Refresh the annotations index so that recently indexed annotation docs are visible.
-        client().admin().indices().prepareRefresh(AnnotationIndex.INDEX_NAME)
+        client().admin().indices().prepareRefresh(AnnotationIndex.LATEST_INDEX_NAME)
             .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
             .execute()
             .actionGet();
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
index 437217d3657bc..6d1ddc1a10d08 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
@@ -7,6 +7,10 @@
 package org.elasticsearch.xpack.ml.integration;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
@@ -14,6 +18,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
@@ -26,11 +31,12 @@
 import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex;
 import org.elasticsearch.xpack.ml.MlSingleNodeTestCase;
 import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor;
-import org.junit.Before;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.is;
 
 public class AnnotationIndexIT extends MlSingleNodeTestCase {
@@ -44,11 +50,6 @@ protected Settings nodeSettings()  {
         return newSettings.build();
     }
 
-    @Before
-    public void createComponents() throws Exception {
-        waitForMlTemplates();
-    }
-
     public void testNotCreatedWhenNoOtherMlIndices() {
 
         // Ask a few times to increase the chance of failure if the .ml-annotations index is created when no other ML index exists
@@ -71,6 +72,52 @@ public void testCreatedWhenAfterOtherMlIndex() throws Exception {
         });
     }
 
+    public void testAliasesMovedFromOldToNew() throws Exception {
+
+        // Create an old annotations index with both read and write aliases pointing at it.
+        String oldIndex = randomFrom(AnnotationIndex.OLD_INDEX_NAMES);
+        CreateIndexRequest createIndexRequest =
+            new CreateIndexRequest(oldIndex)
+                .mapping(AnnotationIndex.annotationsMapping())
+                .settings(Settings.builder()
+                    .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1")
+                    .put(IndexMetadata.SETTING_INDEX_HIDDEN, true))
+                .alias(new Alias(AnnotationIndex.READ_ALIAS_NAME).isHidden(true))
+                .alias(new Alias(AnnotationIndex.WRITE_ALIAS_NAME).isHidden(true));
+        client().execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
+
+        // Because the old annotations index name began with .ml, it will trigger the new annotations index to be created.
+        // When this happens the read alias should be changed to cover both indices, and the write alias should be
+        // switched to only point at the new index.
+        assertBusy(() -> {
+            assertTrue(annotationsIndexExists());
+            ImmutableOpenMap<String, List<AliasMetadata>> aliases = client().admin().indices()
+                .prepareGetAliases(AnnotationIndex.READ_ALIAS_NAME, AnnotationIndex.WRITE_ALIAS_NAME)
+                .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
+                .get()
+                .getAliases();
+            assertNotNull(aliases);
+            List<String> indicesWithReadAlias = new ArrayList<>();
+            for (ObjectObjectCursor<String, List<AliasMetadata>> entry : aliases) {
+                for (AliasMetadata aliasMetadata : entry.value) {
+                    switch (aliasMetadata.getAlias()) {
+                        case AnnotationIndex.WRITE_ALIAS_NAME:
+                            assertThat(entry.key, is(AnnotationIndex.LATEST_INDEX_NAME));
+                            break;
+                        case AnnotationIndex.READ_ALIAS_NAME:
+                            indicesWithReadAlias.add(entry.key);
+                            break;
+                        default:
+                            fail("Found unexpected alias " + aliasMetadata.getAlias() + " on index " + entry.key);
+                            break;
+                    }
+                }
+            }
+            assertThat(indicesWithReadAlias, containsInAnyOrder(oldIndex, AnnotationIndex.LATEST_INDEX_NAME));
+        });
+    }
+
     public void testNotCreatedWhenAfterOtherMlIndexAndUpgradeInProgress() throws Exception {
 
         client().execute(SetUpgradeModeAction.INSTANCE, new SetUpgradeModeAction.Request(true)).actionGet();
@@ -123,7 +170,7 @@ public void testNotCreatedWhenAfterOtherMlIndexAndResetInProgress() throws Excep
     }
 
     private boolean annotationsIndexExists() {
-        return ESIntegTestCase.indexExists(AnnotationIndex.INDEX_NAME, client());
+        return ESIntegTestCase.indexExists(AnnotationIndex.LATEST_INDEX_NAME, client());
     }
 
     private int numberOfAnnotationsAliases() {
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
index 4b23426a2c675..be7568aa18499 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
@@ -731,7 +731,7 @@ private QueryPage<ModelSnapshot> getModelSnapshots() throws Exception {
 
     private List<Annotation> getAnnotations() throws Exception {
         // Refresh the annotations index so that recently indexed annotation docs are visible.
-        client().admin().indices().prepareRefresh(AnnotationIndex.INDEX_NAME)
+        client().admin().indices().prepareRefresh(AnnotationIndex.LATEST_INDEX_NAME)
             .setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED)
             .execute()
             .actionGet();
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
index 28b0a7f1bf672..b65b71b6b7a0a 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
@@ -11,7 +11,9 @@
 import static org.elasticsearch.test.NodeRoles.onlyRole;
 import static org.elasticsearch.test.NodeRoles.removeRoles;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.notNullValue;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -251,7 +253,7 @@ public void testDedicatedMlNode() throws Exception {
             PersistentTask<?> task = tasks.getTask(MlTasks.jobTaskId(jobId));
 
             DiscoveryNode node = clusterState.nodes().resolveNode(task.getExecutorNode());
-            assertThat(node.getAttributes(), hasEntry(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "512"));
+            assertThat(node.getAttributes(), hasEntry(equalTo(MachineLearning.MACHINE_MEMORY_NODE_ATTR), notNullValue()));
             JobTaskState jobTaskState = (JobTaskState) task.getState();
             assertNotNull(jobTaskState);
             assertEquals(JobState.OPENED, jobTaskState.getState());
@@ -513,7 +515,7 @@ private void assertJobTask(String jobId, JobState expectedState, boolean hasExec
             assertNotNull(task.getExecutorNode());
             assertFalse(needsReassignment(task.getAssignment(), clusterState.nodes()));
             DiscoveryNode node = clusterState.nodes().resolveNode(task.getExecutorNode());
-            assertThat(node.getAttributes(), hasEntry(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "512"));
+            assertThat(node.getAttributes(), hasEntry(equalTo(MachineLearning.MACHINE_MEMORY_NODE_ATTR), notNullValue()));
 
             JobTaskState jobTaskState = (JobTaskState) task.getState();
             assertNotNull(jobTaskState);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index d28d66575a0a7..7cfda17f2bdc7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -483,9 +483,8 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
         return Collections.singletonMap(InferenceProcessor.TYPE, inferenceFactory);
     }
 
-    // This is not used in v7 and higher, but users are still prevented from setting it directly to avoid confusion
-    private static final String PRE_V7_ML_ENABLED_NODE_ATTR = "ml.enabled";
-    public static final String MAX_OPEN_JOBS_NODE_ATTR = "ml.max_open_jobs";
+    // This is not used in v8 and higher, but users are still prevented from setting it directly to avoid confusion
+    private static final String PRE_V8_MAX_OPEN_JOBS_NODE_ATTR = "ml.max_open_jobs";
     public static final String MACHINE_MEMORY_NODE_ATTR = "ml.machine_memory";
     public static final String MAX_JVM_SIZE_NODE_ATTR = "ml.max_jvm_size";
     public static final Setting<Integer> CONCURRENT_JOB_ALLOCATIONS =
@@ -622,7 +621,7 @@ public MachineLearning(Settings settings, Path configPath) {
     public static boolean isMlNode(DiscoveryNode node) {
         Map<String, String> nodeAttributes = node.getAttributes();
         try {
-            return Integer.parseInt(nodeAttributes.get(MAX_OPEN_JOBS_NODE_ATTR)) > 0;
+            return Long.parseLong(nodeAttributes.get(MACHINE_MEMORY_NODE_ATTR)) > 0;
         } catch (NumberFormatException e) {
             return false;
         }
@@ -652,29 +651,24 @@ public List<Setting<?>> getSettings() {
     }
 
     public Settings additionalSettings() {
-        String mlEnabledNodeAttrName = "node.attr." + PRE_V7_ML_ENABLED_NODE_ATTR;
-        String maxOpenJobsPerNodeNodeAttrName = "node.attr." + MAX_OPEN_JOBS_NODE_ATTR;
+        String maxOpenJobsPerNodeNodeAttrName = "node.attr." + PRE_V8_MAX_OPEN_JOBS_NODE_ATTR;
         String machineMemoryAttrName = "node.attr." + MACHINE_MEMORY_NODE_ATTR;
         String jvmSizeAttrName = "node.attr." + MAX_JVM_SIZE_NODE_ATTR;
 
         if (enabled == false) {
-            disallowMlNodeAttributes(mlEnabledNodeAttrName, maxOpenJobsPerNodeNodeAttrName, machineMemoryAttrName);
+            disallowMlNodeAttributes(maxOpenJobsPerNodeNodeAttrName, machineMemoryAttrName, jvmSizeAttrName);
             return Settings.EMPTY;
         }
 
         Settings.Builder additionalSettings = Settings.builder();
         if (DiscoveryNode.hasRole(settings, DiscoveryNodeRole.ML_ROLE)) {
-            // TODO: stop setting this attribute in 8.0.0 but disallow it (like mlEnabledNodeAttrName below)
-            // The ML UI will need to be changed to check machineMemoryAttrName instead before this is done
-            addMlNodeAttribute(additionalSettings, maxOpenJobsPerNodeNodeAttrName,
-                    String.valueOf(MAX_OPEN_JOBS_PER_NODE.get(settings)));
             addMlNodeAttribute(additionalSettings, machineMemoryAttrName,
                     Long.toString(OsProbe.getInstance().osStats().getMem().getAdjustedTotal().getBytes()));
             addMlNodeAttribute(additionalSettings, jvmSizeAttrName, Long.toString(Runtime.getRuntime().maxMemory()));
-            // This is not used in v7 and higher, but users are still prevented from setting it directly to avoid confusion
-            disallowMlNodeAttributes(mlEnabledNodeAttrName);
+            // This is not used in v8 and higher, but users are still prevented from setting it directly to avoid confusion
+            disallowMlNodeAttributes(maxOpenJobsPerNodeNodeAttrName);
         } else {
-            disallowMlNodeAttributes(mlEnabledNodeAttrName,
+            disallowMlNodeAttributes(
                 maxOpenJobsPerNodeNodeAttrName,
                 machineMemoryAttrName,
                 jvmSizeAttrName
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
index b4b6b652b5261..d8754b447a1eb 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
@@ -209,7 +209,7 @@ private List<MlDataRemover> createDataRemovers(OriginSettingClient client,
             new ExpiredForecastsRemover(client, threadPool, parentTaskId),
             new ExpiredModelSnapshotsRemover(client,
                 new WrappedBatchedJobsIterator(new SearchAfterJobsIterator(client)), threadPool, parentTaskId, jobResultsProvider, auditor),
-            new UnusedStateRemover(client, clusterService, parentTaskId),
+            new UnusedStateRemover(client, parentTaskId),
             new EmptyStateIndexRemover(client, parentTaskId),
             new UnusedStatsRemover(client, parentTaskId),
             new ExpiredAnnotationsRemover(client,
@@ -226,7 +226,7 @@ private List<MlDataRemover> createDataRemovers(List<Job> jobs, TaskId parentTask
                 threadPool, parentTaskId,
                 jobResultsProvider,
                 auditor),
-            new UnusedStateRemover(client, clusterService, parentTaskId),
+            new UnusedStateRemover(client, parentTaskId),
             new EmptyStateIndexRemover(client, parentTaskId),
             new UnusedStatsRemover(client, parentTaskId),
             new ExpiredAnnotationsRemover(client, new VolatileCursorIterator<>(jobs), parentTaskId, auditor, threadPool)
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java
index 682fab432fbf6..f6d260c61fd2f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
@@ -35,6 +34,7 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAllocationAction;
@@ -161,7 +161,8 @@ protected void masterOperation(Task task, StartTrainedModelDeploymentAction.Requ
                             trainedModelConfig.getModelId(),
                             modelBytes,
                             request.getInferenceThreads(),
-                            request.getModelThreads()
+                            request.getModelThreads(),
+                            request.getQueueCapacity()
                         );
                         PersistentTasksCustomMetadata persistentTasks = clusterService.state().getMetadata().custom(
                             PersistentTasksCustomMetadata.TYPE);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java
index c7ed6d2a88557..b61e3ce55a6cd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java
@@ -425,7 +425,6 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider
         boolean nodeIsMemoryAccurate = true;
         for (DiscoveryNode node : nodes) {
             NodeLoad nodeLoad = nodeLoadDetector.detectNodeLoad(clusterState,
-                true,
                 node,
                 maxOpenJobs,
                 maxMachineMemoryPercent,
@@ -859,7 +858,6 @@ Optional<NativeMemoryCapacity> calculateFutureAvailableCapacity(PersistentTasksC
         Map<String, Long> freeMemoryByNodeId = new HashMap<>();
         for (DiscoveryNode node : mlNodes) {
             NodeLoad nodeLoad = nodeLoadDetector.detectNodeLoad(clusterState,
-                true,
                 node,
                 maxOpenJobs,
                 maxMachineMemoryPercent,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java
index c286650ebcf34..56b5b4c45d5da 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java
@@ -444,7 +444,7 @@ static boolean shouldAllocateModels(final ClusterChangedEvent event) {
     }
 
     Optional<String> nodeHasCapacity(ClusterState state, StartTrainedModelDeploymentAction.TaskParams params, DiscoveryNode node) {
-        NodeLoad load = nodeLoadDetector.detectNodeLoad(state, true, node, maxOpenJobs, maxMemoryPercentage, useAuto);
+        NodeLoad load = nodeLoadDetector.detectNodeLoad(state, node, maxOpenJobs, maxMemoryPercentage, useAuto);
         return handleNodeLoad(load, node.getId(), params);
     }
 
@@ -460,7 +460,6 @@ Optional<String> nodeHasCapacity(
         NodeLoad load = nodeLoadDetector.detectNodeLoad(
             state,
             builder.build(),
-            true,
             node,
             maxOpenJobs,
             maxMemoryPercentage,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationNodeService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationNodeService.java
index 6db99931d8c46..a40aad7a7f075 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationNodeService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationNodeService.java
@@ -332,6 +332,8 @@ TrainedModelDeploymentTask getTask(String modelId) {
     }
 
     void prepareModelToLoad(StartTrainedModelDeploymentAction.TaskParams taskParams) {
+        logger.debug(() -> new ParameterizedMessage("[{}] preparing to load model with task params: {}",
+            taskParams.getModelId(), taskParams));
         TrainedModelDeploymentTask task = (TrainedModelDeploymentTask) taskManager.register(
             TRAINED_MODEL_ALLOCATION_TASK_TYPE,
             TRAINED_MODEL_ALLOCATION_TASK_NAME_PREFIX + taskParams.getModelId(),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java
index 8ae0aeb3f4dde..c19c0b8776179 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java
@@ -392,7 +392,11 @@ class ProcessContext {
             this.task = Objects.requireNonNull(task);
             resultProcessor = new PyTorchResultProcessor(task.getModelId());
             this.stateStreamer = new PyTorchStateStreamer(client, executorService, xContentRegistry);
-            this.executorService = new ProcessWorkerExecutorService(threadPool.getThreadContext(), "pytorch_inference", 1024);
+            this.executorService = new ProcessWorkerExecutorService(
+                threadPool.getThreadContext(),
+                "pytorch_inference",
+                task.getParams().getQueueCapacity()
+            );
         }
 
         PyTorchResultProcessor getResultProcessor() {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java
index 56bdedb74fc60..42a9c3cf90703 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java
@@ -115,7 +115,6 @@ public Tuple<NativeMemoryCapacity, Long> perceivedCapacityAndMaxFreeMemory(int m
         long mostAvailableMemory = capableNodes.stream()
             .map(n -> nodeLoadDetector.detectNodeLoad(
                 clusterState,
-                true,
                 n,
                 maxOpenJobs,
                 maxMachineMemoryPercent,
@@ -171,7 +170,6 @@ public PersistentTasksCustomMetadata.Assignment selectNode(Long estimatedMemoryF
             }
             NodeLoad currentLoad = nodeLoadDetector.detectNodeLoad(
                 clusterState,
-                true, // Remove in 8.0.0
                 node,
                 dynamicMaxOpenJobs,
                 maxMachineMemoryPercent,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/NodeLoadDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/NodeLoadDetector.java
index 64f85fe71f505..f160b0dcb921f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/NodeLoadDetector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/NodeLoadDetector.java
@@ -43,7 +43,6 @@ public MlMemoryTracker getMlMemoryTracker() {
     }
 
     public NodeLoad detectNodeLoad(ClusterState clusterState,
-                                   boolean allNodesHaveDynamicMaxWorkers,
                                    DiscoveryNode node,
                                    int dynamicMaxOpenJobs,
                                    int maxMachineMemoryPercent,
@@ -51,7 +50,6 @@ public NodeLoad detectNodeLoad(ClusterState clusterState,
         return detectNodeLoad(
             clusterState,
             TrainedModelAllocationMetadata.fromState(clusterState),
-            allNodesHaveDynamicMaxWorkers,
             node,
             dynamicMaxOpenJobs,
             maxMachineMemoryPercent,
@@ -61,25 +59,13 @@ public NodeLoad detectNodeLoad(ClusterState clusterState,
 
     public NodeLoad detectNodeLoad(ClusterState clusterState,
                                    TrainedModelAllocationMetadata allocationMetadata,
-                                   boolean allNodesHaveDynamicMaxWorkers,
                                    DiscoveryNode node,
-                                   int dynamicMaxOpenJobs,
+                                   int maxNumberOfOpenJobs,
                                    int maxMachineMemoryPercent,
                                    boolean useAutoMachineMemoryCalculation) {
         PersistentTasksCustomMetadata persistentTasks = clusterState.getMetadata().custom(PersistentTasksCustomMetadata.TYPE);
         Map<String, String> nodeAttributes = node.getAttributes();
         List<String> errors = new ArrayList<>();
-        int maxNumberOfOpenJobs = dynamicMaxOpenJobs;
-        // TODO: remove this in 8.0.0
-        if (allNodesHaveDynamicMaxWorkers == false) {
-            String maxNumberOfOpenJobsStr = nodeAttributes.get(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR);
-            try {
-                maxNumberOfOpenJobs = Integer.parseInt(maxNumberOfOpenJobsStr);
-            } catch (NumberFormatException e) {
-                errors.add(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR + " attribute [" + maxNumberOfOpenJobsStr + "] is not an integer");
-                maxNumberOfOpenJobs = -1;
-            }
-        }
         OptionalLong maxMlMemory = NativeMemoryCalculator.allowedBytesForMl(node,
             maxMachineMemoryPercent,
             useAutoMachineMemoryCalculation);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessWorkerExecutorService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessWorkerExecutorService.java
index f66018793a03c..ffa35a1549aaa 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessWorkerExecutorService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessWorkerExecutorService.java
@@ -45,14 +45,14 @@ public class ProcessWorkerExecutorService extends AbstractExecutorService {
     /**
      * @param contextHolder the thread context holder
      * @param processName the name of the process to be used in logging
-     * @param queueSize the size of the queue holding operations. If an operation is added
+     * @param queueCapacity the capacity of the queue holding operations. If an operation is added
      *                  for execution when the queue is full a 429 error is thrown.
      */
     @SuppressForbidden(reason = "properly rethrowing errors, see EsExecutors.rethrowErrors")
-    public ProcessWorkerExecutorService(ThreadContext contextHolder, String processName, int queueSize) {
+    public ProcessWorkerExecutorService(ThreadContext contextHolder, String processName, int queueCapacity) {
         this.contextHolder = Objects.requireNonNull(contextHolder);
         this.processName = Objects.requireNonNull(processName);
-        this.queue = new LinkedBlockingQueue<>(queueSize);
+        this.queue = new LinkedBlockingQueue<>(queueCapacity);
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java
index 42eee2eac8157..22ea0dbcf9234 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java
@@ -11,14 +11,12 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.OriginSettingClient;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xpack.core.ml.MlConfigIndex;
-import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
@@ -52,13 +50,10 @@ public class UnusedStateRemover implements MlDataRemover {
     private static final Logger LOGGER = LogManager.getLogger(UnusedStateRemover.class);
 
     private final OriginSettingClient client;
-    private final ClusterService clusterService;
     private final TaskId parentTaskId;
 
-    public UnusedStateRemover(OriginSettingClient client, ClusterService clusterService,
-                              TaskId parentTaskId) {
+    public UnusedStateRemover(OriginSettingClient client, TaskId parentTaskId) {
         this.client = Objects.requireNonNull(client);
-        this.clusterService = Objects.requireNonNull(clusterService);
         this.parentTaskId = Objects.requireNonNull(parentTaskId);
     }
 
@@ -111,10 +106,6 @@ private Set<String> getJobIds() {
     private Set<String> getAnomalyDetectionJobIds() {
         Set<String> jobIds = new HashSet<>();
 
-        // TODO Once at 8.0, we can stop searching for jobs in cluster state
-        // and remove cluster service as a member all together.
-        jobIds.addAll(MlMetadata.getMlMetadata(clusterService.state()).getJobs().keySet());
-
         DocIdBatchedDocumentIterator iterator = new DocIdBatchedDocumentIterator(client, MlConfigIndex.indexName(),
             QueryBuilders.termQuery(Job.JOB_TYPE.getPreferredName(), Job.ANOMALY_DETECTOR_JOB_TYPE));
         while (iterator.hasNext()) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java
index 6fa6405b461b0..f3cf06bf70684 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java
@@ -23,6 +23,7 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction.Request.INFERENCE_THREADS;
 import static org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction.Request.MODEL_THREADS;
+import static org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction.Request.QUEUE_CAPACITY;
 import static org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction.Request.TIMEOUT;
 import static org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction.Request.WAIT_FOR;
 
@@ -59,6 +60,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
             ));
             request.setInferenceThreads(restRequest.paramAsInt(INFERENCE_THREADS.getPreferredName(), request.getInferenceThreads()));
             request.setModelThreads(restRequest.paramAsInt(MODEL_THREADS.getPreferredName(), request.getModelThreads()));
+            request.setQueueCapacity(restRequest.paramAsInt(QUEUE_CAPACITY.getPreferredName(), request.getQueueCapacity()));
         }
 
         return channel -> client.execute(StartTrainedModelDeploymentAction.INSTANCE, request, new RestToXContentListener<>(channel));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java
index f00da807175a8..9a283f31abe6e 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java
@@ -542,8 +542,7 @@ private void givenJobs(List<Job> jobs, List<GetJobsStatsAction.Response.JobStats
     private ClusterState givenNodeCount(int nodeCount) {
         DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder();
         for (int i = 0; i < nodeCount; i++) {
-            Map<String, String> attrs = new HashMap<>();
-            attrs.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(20));
+            Map<String, String> attrs = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
             Set<DiscoveryNodeRole> roles = new HashSet<>();
             roles.add(DiscoveryNodeRole.DATA_ROLE);
             roles.add(DiscoveryNodeRole.MASTER_ROLE);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java
index 3785d6f89262c..f33191afd6b5f 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java
@@ -164,11 +164,7 @@ public void testNoAttributes_givenSameAndMlEnabled() {
 
     public void testNoAttributes_givenClash() {
         Settings.Builder builder = Settings.builder();
-        if (randomBoolean()) {
-            builder.put("node.attr.ml.enabled", randomBoolean());
-        } else {
-            builder.put("node.attr.ml.max_open_jobs", randomIntBetween(13, 15));
-        }
+        builder.put("node.attr.ml.max_open_jobs", randomIntBetween(13, 15));
         MachineLearning machineLearning = createMachineLearning(builder.put("path.home", createTempDir()).build());
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, machineLearning::additionalSettings);
         assertThat(e.getMessage(), startsWith("Directly setting [node.attr.ml."));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java
index 14253cffff2a9..5a4f3c3d0d365 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java
@@ -176,7 +176,7 @@ private static DiscoveryNode createNode(int i, boolean isMlNode, Version nodeVer
             "_node_name" + i,
             "_node_id" + i,
             new TransportAddress(InetAddress.getLoopbackAddress(), 9300 + i),
-            Map.of("ml.max_open_jobs", isMlNode ? "10" : "0", "ml.machine_memory", String.valueOf(ByteSizeValue.ofGb(1).getBytes())),
+            isMlNode ? Map.of("ml.machine_memory", String.valueOf(ByteSizeValue.ofGb(1).getBytes())) : Map.of(),
             Collections.emptySet(),
             nodeVersion);
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java
index 5de8e1885fe78..6a7c7eb892a95 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java
@@ -88,7 +88,7 @@ public void setup() {
         when(mlMemoryTracker.getTrainedModelAllocationMemoryRequirement(any())).thenReturn(DEFAULT_JOB_SIZE);
         nodeLoadDetector = mock(NodeLoadDetector.class);
         when(nodeLoadDetector.getMlMemoryTracker()).thenReturn(mlMemoryTracker);
-        when(nodeLoadDetector.detectNodeLoad(any(), anyBoolean(), any(), anyInt(), anyInt(), anyBoolean()))
+        when(nodeLoadDetector.detectNodeLoad(any(), any(), anyInt(), anyInt(), anyBoolean()))
             .thenReturn(NodeLoad.builder("any")
                 .setUseMemory(true)
                 .incAssignedJobMemory(ByteSizeValue.ofGb(1).getBytes())
@@ -704,9 +704,8 @@ private static List<DiscoveryNode> withMlNodes(String... nodeName) {
                 MapBuilder.<String, String>newMapBuilder()
                     .put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, String.valueOf(DEFAULT_NODE_SIZE))
                     .put(MachineLearning.MAX_JVM_SIZE_NODE_ATTR, String.valueOf(DEFAULT_JVM_SIZE))
-                    .put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, String.valueOf(10))
                     .map(),
-                new HashSet<>(Arrays.asList(DiscoveryNodeRole.MASTER_ROLE)),
+                new HashSet<>(List.of(DiscoveryNodeRole.MASTER_ROLE)),
                 Version.CURRENT))
             .collect(Collectors.toList());
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterServiceTests.java
index 0c974d17fbce1..e6b82316b9663 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterServiceTests.java
@@ -940,7 +940,7 @@ private static DiscoveryNode buildOldNode(String name, boolean isML, long native
     }
 
     private static StartTrainedModelDeploymentAction.TaskParams newParams(String modelId, long modelSize) {
-        return new StartTrainedModelDeploymentAction.TaskParams(modelId, modelSize, 1, 1);
+        return new StartTrainedModelDeploymentAction.TaskParams(modelId, modelSize, 1, 1, 1024);
     }
 
     private static void assertNodeState(TrainedModelAllocationMetadata metadata, String modelId, String nodeId, RoutingState routingState) {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationMetadataTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationMetadataTests.java
index 1de97cc5991f3..ccb9b27be591e 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationMetadataTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationMetadataTests.java
@@ -99,7 +99,8 @@ private static StartTrainedModelDeploymentAction.TaskParams randomParams(String
             modelId,
             randomNonNegativeLong(),
             randomIntBetween(1, 8),
-            randomIntBetween(1, 8)
+            randomIntBetween(1, 8),
+            randomIntBetween(1, 10000)
         );
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationNodeServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationNodeServiceTests.java
index 88cd412c30147..0b8bffdf302aa 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationNodeServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationNodeServiceTests.java
@@ -497,7 +497,7 @@ private void withSearchingLoadFailure(String modelId) {
     }
 
     private static StartTrainedModelDeploymentAction.TaskParams newParams(String modelId) {
-        return new StartTrainedModelDeploymentAction.TaskParams(modelId, randomNonNegativeLong(), 1, 1);
+        return new StartTrainedModelDeploymentAction.TaskParams(modelId, randomNonNegativeLong(), 1, 1, 1024);
     }
 
     private TrainedModelAllocationNodeService createService() {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java
index 104810e33c022..8662a0f42b8e0 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java
@@ -54,7 +54,6 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-// TODO: in 8.0.0 remove all instances of MAX_OPEN_JOBS_NODE_ATTR from this file
 public class JobNodeSelectorTests extends ESTestCase {
 
     // To simplify the logic in this class all jobs have the same memory requirement
@@ -103,9 +102,7 @@ public void testSelectLeastLoadedMlNodeForAnomalyDetectorJob_maxCapacityCountLim
         int maxMachineMemoryPercent = 30;
         long machineMemory = (maxRunningJobsPerNode + 1) * JOB_MEMORY_REQUIREMENT.getBytes() * 100 / maxMachineMemoryPercent;
 
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
 
         ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, maxRunningJobsPerNode);
 
@@ -134,9 +131,7 @@ public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_maxCapacityCount
         int maxMachineMemoryPercent = 30;
         long machineMemory = (maxRunningJobsPerNode + 1) * JOB_MEMORY_REQUIREMENT.getBytes() * 100 / maxMachineMemoryPercent;
 
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
 
         ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, maxRunningJobsPerNode);
 
@@ -171,9 +166,7 @@ public void testSelectLeastLoadedMlNodeForAnomalyDetectorJob_maxCapacityMemoryLi
             currentlyRunningJobsPerNode * JOB_MEMORY_REQUIREMENT.getBytes();
         long machineMemory = currentlyRunningJobMemory * 100 / maxMachineMemoryPercent;
 
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
 
         ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, currentlyRunningJobsPerNode);
 
@@ -204,9 +197,7 @@ public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_givenTaskHasNull
         int maxRunningJobsPerNode = 10;
         int maxMachineMemoryPercent = 30;
 
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, String.valueOf(ByteSizeValue.ofGb(1).getBytes()));
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, String.valueOf(ByteSizeValue.ofGb(1).getBytes()));
 
         ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, 1, JobState.OPENED, null);
 
@@ -233,9 +224,7 @@ public void testSelectLeastLoadedMlNodeForAnomalyDetectorJob_firstJobTooBigMemor
         long firstJobTotalMemory = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes() + JOB_MEMORY_REQUIREMENT.getBytes();
         long machineMemory = (firstJobTotalMemory - 1) * 100 / maxMachineMemoryPercent;
 
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
 
         ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, 0);
 
@@ -271,9 +260,7 @@ public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_maxCapacityMemor
             currentlyRunningJobsPerNode * JOB_MEMORY_REQUIREMENT.getBytes();
         long machineMemory = currentlyRunningJobMemory * 100 / maxMachineMemoryPercent;
 
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
 
         ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, currentlyRunningJobsPerNode);
 
@@ -306,9 +293,7 @@ public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_firstJobTooBigMe
         long firstJobTotalMemory = MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes() + JOB_MEMORY_REQUIREMENT.getBytes();
         long machineMemory = (firstJobTotalMemory - 1) * 100 / maxMachineMemoryPercent;
 
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
 
         ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, 0);
 
@@ -370,9 +355,7 @@ public void testSelectLeastLoadedMlNode_noMlNodes() {
     }
 
     public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
         DiscoveryNodes nodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
                 nodeAttr, Collections.emptySet(), Version.CURRENT))
@@ -479,9 +462,7 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() {
     }
 
     public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
         DiscoveryNodes nodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
                 nodeAttr, Collections.emptySet(), Version.CURRENT))
@@ -548,9 +529,7 @@ public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob()
     }
 
     public void testSelectLeastLoadedMlNode_noCompatibleJobTypeNodes() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
         DiscoveryNodes nodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
                 nodeAttr, Collections.emptySet(), Version.CURRENT))
@@ -590,9 +569,7 @@ public void testSelectLeastLoadedMlNode_noCompatibleJobTypeNodes() {
     }
 
     public void testSelectLeastLoadedMlNode_reasonsAreInDeterministicOrder() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
         DiscoveryNodes nodes = DiscoveryNodes.builder()
             .add(
                 new DiscoveryNode(
@@ -659,9 +636,7 @@ public void testSelectLeastLoadedMlNode_reasonsAreInDeterministicOrder() {
     }
 
     public void testSelectLeastLoadedMlNode_noNodesMatchingModelSnapshotMinVersion() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
         DiscoveryNodes nodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
                 nodeAttr, Collections.emptySet(), Version.fromString("6.2.0")))
@@ -699,9 +674,7 @@ public void testSelectLeastLoadedMlNode_noNodesMatchingModelSnapshotMinVersion()
     }
 
     public void testSelectLeastLoadedMlNode_jobWithRules() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
         DiscoveryNodes nodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
                 nodeAttr, Collections.emptySet(), Version.fromString("6.2.0")))
@@ -735,9 +708,7 @@ public void testSelectLeastLoadedMlNode_jobWithRules() {
     }
 
     public void testSelectMlNodeOnlyOutOfCandidates() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
         DiscoveryNodes nodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
                 nodeAttr, Collections.emptySet(), Version.CURRENT))
@@ -825,9 +796,7 @@ public void testMaximumPossibleNodeMemoryTooSmall() {
         int maxMachineMemoryPercent = 30;
         long machineMemory = (maxRunningJobsPerNode + 1) * JOB_MEMORY_REQUIREMENT.getBytes() * 100 / maxMachineMemoryPercent;
 
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
+        Map<String, String> nodeAttr = Map.of(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
 
         ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, maxRunningJobsPerNode);
 
@@ -860,7 +829,6 @@ public void testPerceivedCapacityAndMaxFreeMemory() {
                 "filled_ml_node_id",
                 new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
                 MapBuilder.<String, String>newMapBuilder()
-                    .put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "1")
                     .put(MachineLearning.MAX_JVM_SIZE_NODE_ATTR, "10")
                     .put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(ByteSizeValue.ofGb(30).getBytes()))
                     .map(),
@@ -870,7 +838,6 @@ public void testPerceivedCapacityAndMaxFreeMemory() {
                 "not_filled_ml_node_id",
                 new TransportAddress(InetAddress.getLoopbackAddress(), 9302),
                 MapBuilder.<String, String>newMapBuilder()
-                    .put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10")
                     .put(MachineLearning.MAX_JVM_SIZE_NODE_ATTR, "10")
                     .put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(ByteSizeValue.ofGb(30).getBytes()))
                     .map(),
@@ -880,7 +847,6 @@ public void testPerceivedCapacityAndMaxFreeMemory() {
                 "not_filled_smaller_ml_node_id",
                 new TransportAddress(InetAddress.getLoopbackAddress(), 9303),
                 MapBuilder.<String, String>newMapBuilder()
-                    .put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10")
                     .put(MachineLearning.MAX_JVM_SIZE_NODE_ATTR, "10")
                     .put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(ByteSizeValue.ofGb(10).getBytes()))
                     .map(),
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/NodeLoadDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/NodeLoadDetectorTests.java
index 8ef497c4aa956..c99758f27008c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/NodeLoadDetectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/NodeLoadDetectorTests.java
@@ -38,7 +38,6 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-// TODO: in 8.0.0 remove all instances of MAX_OPEN_JOBS_NODE_ATTR from this file
 public class NodeLoadDetectorTests extends ESTestCase {
 
     // To simplify the logic in this class all jobs have the same memory requirement
@@ -60,7 +59,6 @@ public void setup() {
 
     public void testNodeLoadDetection() {
         Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
         nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "-1");
         // MachineLearning.MACHINE_MEMORY_NODE_ATTR negative, so this will fall back to allocating by count
         DiscoveryNodes nodes = DiscoveryNodes.builder()
@@ -91,7 +89,8 @@ public void testNodeLoadDetection() {
                                 .addNewAllocation(
                                     "model1",
                                     TrainedModelAllocation.Builder
-                                        .empty(new StartTrainedModelDeploymentAction.TaskParams("model1", MODEL_MEMORY_REQUIREMENT, 1, 1))
+                                        .empty(new StartTrainedModelDeploymentAction.TaskParams(
+                                            "model1", MODEL_MEMORY_REQUIREMENT, 1, 1, 1024))
                                         .addNewRoutingEntry("_node_id4")
                                         .addNewFailedRoutingEntry("_node_id2", "test")
                                         .addNewRoutingEntry("_node_id1")
@@ -107,28 +106,28 @@ public void testNodeLoadDetection() {
                         )
                 ).build();
 
-        NodeLoad load = nodeLoadDetector.detectNodeLoad(cs, true, nodes.get("_node_id1"), 10, 30, false);
+        NodeLoad load = nodeLoadDetector.detectNodeLoad(cs, nodes.get("_node_id1"), 10, 30, false);
         assertThat(load.getAssignedJobMemory(), equalTo(52428800L));
         assertThat(load.getNumAllocatingJobs(), equalTo(2L));
         assertThat(load.getNumAssignedJobs(), equalTo(2L));
         assertThat(load.getMaxJobs(), equalTo(10));
         assertThat(load.getMaxMlMemory(), equalTo(0L));
 
-        load = nodeLoadDetector.detectNodeLoad(cs, true, nodes.get("_node_id2"), 5, 30, false);
+        load = nodeLoadDetector.detectNodeLoad(cs, nodes.get("_node_id2"), 5, 30, false);
         assertThat(load.getAssignedJobMemory(), equalTo(41943040L));
         assertThat(load.getNumAllocatingJobs(), equalTo(1L));
         assertThat(load.getNumAssignedJobs(), equalTo(1L));
         assertThat(load.getMaxJobs(), equalTo(5));
         assertThat(load.getMaxMlMemory(), equalTo(0L));
 
-        load = nodeLoadDetector.detectNodeLoad(cs, true, nodes.get("_node_id3"), 5, 30, false);
+        load = nodeLoadDetector.detectNodeLoad(cs, nodes.get("_node_id3"), 5, 30, false);
         assertThat(load.getAssignedJobMemory(), equalTo(0L));
         assertThat(load.getNumAllocatingJobs(), equalTo(0L));
         assertThat(load.getNumAssignedJobs(), equalTo(0L));
         assertThat(load.getMaxJobs(), equalTo(5));
         assertThat(load.getMaxMlMemory(), equalTo(0L));
 
-        load = nodeLoadDetector.detectNodeLoad(cs, true, nodes.get("_node_id4"), 5, 30, false);
+        load = nodeLoadDetector.detectNodeLoad(cs, nodes.get("_node_id4"), 5, 30, false);
         assertThat(load.getAssignedJobMemory(), equalTo(429916160L));
         assertThat(load.getNumAllocatingJobs(), equalTo(0L));
         assertThat(load.getNumAssignedJobs(), equalTo(2L));
@@ -136,27 +135,8 @@ public void testNodeLoadDetection() {
         assertThat(load.getMaxMlMemory(), equalTo(0L));
     }
 
-    public void testNodeLoadDetection_withBadMaxOpenJobsAttribute() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "foo");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "-1");
-        DiscoveryNodes nodes = DiscoveryNodes.builder()
-            .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
-                nodeAttr, Collections.emptySet(), Version.CURRENT))
-            .build();
-
-        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
-        cs.nodes(nodes);
-        Metadata.Builder metadata = Metadata.builder();
-        cs.metadata(metadata);
-
-        NodeLoad load = nodeLoadDetector.detectNodeLoad(cs.build(), false, nodes.get("_node_id1"), 10, 30, false);
-        assertThat(load.getError(), containsString("ml.max_open_jobs attribute [foo] is not an integer"));
-    }
-
     public void testNodeLoadDetection_withBadMachineMemoryAttribute() {
         Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
         nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "bar");
         DiscoveryNodes nodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
@@ -168,7 +148,7 @@ public void testNodeLoadDetection_withBadMachineMemoryAttribute() {
         Metadata.Builder metadata = Metadata.builder();
         cs.metadata(metadata);
 
-        NodeLoad load = nodeLoadDetector.detectNodeLoad(cs.build(), false, nodes.get("_node_id1"), 10, -1, false);
+        NodeLoad load = nodeLoadDetector.detectNodeLoad(cs.build(), nodes.get("_node_id1"), 10, -1, false);
         assertThat(load.getError(), containsString("ml.machine_memory attribute [bar] is not a long"));
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
index 50ba3baa09e7f..9881563350c81 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
@@ -92,6 +92,7 @@
 import java.util.function.Consumer;
 
 import static org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
@@ -184,21 +185,23 @@ public void setup() throws Exception {
                             Settings.builder()
                                 .put(SETTING_NUMBER_OF_SHARDS, 1)
                                 .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                                .put(SETTING_INDEX_HIDDEN, true)
                                 .put(SETTING_VERSION_CREATED, Version.CURRENT)
                                 .build())
-                        .putAlias(AliasMetadata.builder(AnomalyDetectorsIndex.jobStateIndexWriteAlias()).build())
+                        .putAlias(AliasMetadata.builder(AnomalyDetectorsIndex.jobStateIndexWriteAlias()).isHidden(true).build())
                         .build())
                 .fPut(
-                    AnnotationIndex.INDEX_NAME,
-                    IndexMetadata.builder(AnnotationIndex.INDEX_NAME)
+                    AnnotationIndex.LATEST_INDEX_NAME,
+                    IndexMetadata.builder(AnnotationIndex.LATEST_INDEX_NAME)
                         .settings(
                             Settings.builder()
                                 .put(SETTING_NUMBER_OF_SHARDS, 1)
                                 .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                                .put(SETTING_INDEX_HIDDEN, true)
                                 .put(SETTING_VERSION_CREATED, Version.CURRENT)
                                 .build())
-                        .putAlias(AliasMetadata.builder(AnnotationIndex.READ_ALIAS_NAME).build())
-                        .putAlias(AliasMetadata.builder(AnnotationIndex.WRITE_ALIAS_NAME).build())
+                        .putAlias(AliasMetadata.builder(AnnotationIndex.READ_ALIAS_NAME).isHidden(true).build())
+                        .putAlias(AliasMetadata.builder(AnnotationIndex.WRITE_ALIAS_NAME).isHidden(true).build())
                         .build())
                 .build())
             .build();
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
index 96f25c319975a..14ece00b640dc 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.monitoring.MonitoringDeprecatedSettings;
 import org.elasticsearch.xpack.core.monitoring.MonitoringField;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringMigrateAlertsAction;
diff --git a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/LinuxPreallocator.java b/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/LinuxPreallocator.java
index 02eb5d0add036..f692a5cb709d9 100644
--- a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/LinuxPreallocator.java
+++ b/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/LinuxPreallocator.java
@@ -16,7 +16,7 @@
 final class LinuxPreallocator implements Preallocator {
 
     @Override
-    public boolean available() {
+    public boolean useNative() {
         return Natives.NATIVES_AVAILABLE;
     }
 
diff --git a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/MacOsPreallocator.java b/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/MacOsPreallocator.java
index 61e7b3eafebf4..65303c5b0c5d6 100644
--- a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/MacOsPreallocator.java
+++ b/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/MacOsPreallocator.java
@@ -20,7 +20,7 @@
 final class MacOsPreallocator implements Preallocator {
 
     @Override
-    public boolean available() {
+    public boolean useNative() {
         return Natives.NATIVES_AVAILABLE;
     }
 
diff --git a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/UnsupportedPreallocator.java b/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/NoNativePreallocator.java
similarity index 87%
rename from x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/UnsupportedPreallocator.java
rename to x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/NoNativePreallocator.java
index 6859824e508f9..329b609cd6768 100644
--- a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/UnsupportedPreallocator.java
+++ b/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/NoNativePreallocator.java
@@ -7,10 +7,10 @@
 
 package org.elasticsearch.xpack.searchablesnapshots.preallocate;
 
-final class UnsupportedPreallocator implements Preallocator {
+final class NoNativePreallocator implements Preallocator {
 
     @Override
-    public boolean available() {
+    public boolean useNative() {
         return false;
     }
 
diff --git a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocate.java b/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocate.java
index ffead10b79335..500c675dc8b0e 100644
--- a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocate.java
+++ b/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocate.java
@@ -11,10 +11,12 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.SuppressForbidden;
 
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.lang.reflect.Field;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -31,35 +33,53 @@ public static void preallocate(final Path cacheFile, final long fileSize) throws
         } else if (Constants.MAC_OS_X) {
             preallocate(cacheFile, fileSize, new MacOsPreallocator());
         } else {
-            preallocate(cacheFile, fileSize, new UnsupportedPreallocator());
+            preallocate(cacheFile, fileSize, new NoNativePreallocator());
         }
     }
 
     @SuppressForbidden(reason = "need access to fd on FileOutputStream")
     private static void preallocate(final Path cacheFile, final long fileSize, final Preallocator prealloactor) throws IOException {
-        if (prealloactor.available() == false) {
-            logger.warn("failed to pre-allocate cache file [{}] as native methods are not available", cacheFile);
-        }
         boolean success = false;
-        try (FileOutputStream fileChannel = new FileOutputStream(cacheFile.toFile())) {
-            long currentSize = fileChannel.getChannel().size();
-            if (currentSize < fileSize) {
-                final Field field = AccessController.doPrivileged(new FileDescriptorFieldAction(fileChannel));
-                final int errno = prealloactor.preallocate((int) field.get(fileChannel.getFD()), currentSize, fileSize - currentSize);
-                if (errno == 0) {
+        try {
+            if (prealloactor.useNative()) {
+                try (FileOutputStream fileChannel = new FileOutputStream(cacheFile.toFile())) {
+                    long currentSize = fileChannel.getChannel().size();
+                    if (currentSize < fileSize) {
+                        logger.info("pre-allocating cache file [{}] ({}) using native methods", cacheFile, new ByteSizeValue(fileSize));
+                        final Field field = AccessController.doPrivileged(new FileDescriptorFieldAction(fileChannel));
+                        final int errno = prealloactor.preallocate(
+                            (int) field.get(fileChannel.getFD()),
+                            currentSize,
+                            fileSize - currentSize
+                        );
+                        if (errno == 0) {
+                            success = true;
+                            logger.debug("pre-allocated cache file [{}] using native methods", cacheFile);
+                        } else {
+                            logger.warn(
+                                "failed to pre-allocate cache file [{}] using native methods, errno: [{}], error: [{}]",
+                                cacheFile,
+                                errno,
+                                prealloactor.error(errno)
+                            );
+                        }
+                    }
+                } catch (final Exception e) {
+                    logger.warn(new ParameterizedMessage("failed to pre-allocate cache file [{}] using native methods", cacheFile), e);
+                }
+            }
+            // even if allocation was successful above, verify again here
+            try (RandomAccessFile raf = new RandomAccessFile(cacheFile.toFile(), "rw")) {
+                if (raf.length() != fileSize) {
+                    logger.info("pre-allocating cache file [{}] ({}) using setLength method", cacheFile, new ByteSizeValue(fileSize));
+                    raf.setLength(fileSize);
                     success = true;
-                    logger.debug("pre-allocated cache file [{}] using native methods", cacheFile);
-                } else {
-                    logger.warn(
-                        "failed to pre-allocate cache file [{}] using native methods, errno: [{}], error: [{}]",
-                        cacheFile,
-                        errno,
-                        prealloactor.error(errno)
-                    );
+                    logger.debug("pre-allocated cache file [{}] using setLength method", cacheFile);
                 }
+            } catch (final Exception e) {
+                logger.warn(new ParameterizedMessage("failed to pre-allocate cache file [{}] using setLength method", cacheFile), e);
+                throw e;
             }
-        } catch (final Exception e) {
-            logger.warn(new ParameterizedMessage("failed to pre-allocate cache file [{}] using native methods", cacheFile), e);
         } finally {
             if (success == false) {
                 // if anything goes wrong, delete the potentially created file to not waste disk space
diff --git a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocator.java b/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocator.java
index 05b181ac122f1..46dafc7352c99 100644
--- a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocator.java
+++ b/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocator.java
@@ -17,7 +17,7 @@ interface Preallocator {
      *
      * @return true if native methods are available, otherwise false
      */
-    boolean available();
+    boolean useNative();
 
     /**
      * Pre-allocate a file of given current size to the specified size using the given file descriptor.
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/SharedBytes.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/SharedBytes.java
index e3f21befa0cc1..967f9ff500efc 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/SharedBytes.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/SharedBytes.java
@@ -9,7 +9,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.io.Channels;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.AbstractRefCounted;
@@ -61,27 +60,8 @@ public class SharedBytes extends AbstractRefCounted {
         if (fileSize > 0) {
             cacheFile = findCacheSnapshotCacheFilePath(environment, fileSize);
             Preallocate.preallocate(cacheFile, fileSize);
-            // TODO: maybe make this faster by allocating a larger direct buffer if this is too slow for very large files
-            // We fill either the full file or the bytes between its current size and the desired size once with zeros to fully allocate
-            // the file up front
-            final ByteBuffer fillBytes = ByteBuffer.allocate(Channels.WRITE_CHUNK_SIZE);
             this.fileChannel = FileChannel.open(cacheFile, OPEN_OPTIONS);
-            long written = fileChannel.size();
-            if (fileSize < written) {
-                logger.info("creating shared snapshot cache file [size={}, path={}]", fileSize, cacheFile);
-            } else if (fileSize == written) {
-                logger.debug("reusing existing shared snapshot cache file [size={}, path={}]", fileSize, cacheFile);
-            }
-            fileChannel.position(written);
-            while (written < fileSize) {
-                final int toWrite = Math.toIntExact(Math.min(fileSize - written, Channels.WRITE_CHUNK_SIZE));
-                fillBytes.position(0).limit(toWrite);
-                Channels.writeToChannel(fillBytes, fileChannel);
-                written += toWrite;
-            }
-            if (written > fileChannel.size()) {
-                fileChannel.truncate(fileSize);
-            }
+            assert this.fileChannel.size() == fileSize : "expected file size " + fileSize + " but was " + fileChannel.size();
         } else {
             this.fileChannel = null;
             for (Path path : environment.nodeDataPaths()) {
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheServiceTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheServiceTests.java
index 7edb666468a37..1f591cdfde463 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheServiceTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheServiceTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.RatioValue;
 import org.elasticsearch.common.unit.RelativeByteSizeValue;
@@ -346,4 +347,33 @@ private static CacheKey generateCacheKey() {
             randomAlphaOfLength(10)
         );
     }
+
+    public void testCacheSizeChanges() throws IOException {
+        ByteSizeValue val1 = new ByteSizeValue(randomIntBetween(1, 5), ByteSizeUnit.MB);
+        Settings settings = Settings.builder()
+            .put(NODE_NAME_SETTING.getKey(), "node")
+            .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), val1.getStringRep())
+            .put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), new ByteSizeValue(size(100)).getStringRep())
+            .put("path.home", createTempDir())
+            .build();
+        final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
+        try (
+            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
+            FrozenCacheService cacheService = new FrozenCacheService(environment, settings, taskQueue.getThreadPool())
+        ) {
+            assertEquals(val1.getBytes(), cacheService.getStats().getSize());
+        }
+
+        ByteSizeValue val2 = new ByteSizeValue(randomIntBetween(1, 5), ByteSizeUnit.MB);
+        settings = Settings.builder()
+            .put(settings)
+            .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), val2.getStringRep())
+            .build();
+        try (
+            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
+            FrozenCacheService cacheService = new FrozenCacheService(environment, settings, taskQueue.getThreadPool())
+        ) {
+            assertEquals(val2.getBytes(), cacheService.getStats().getSize());
+        }
+    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
index 27bf9ac1f21c4..fe9bec298c8bd 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
@@ -222,11 +222,6 @@ ResolvedIndices resolveIndicesAndAliases(String action, IndicesRequest indicesRe
                 }
                 List<String> replaced = indexAbstractionResolver.resolveIndexAbstractions(split.getLocal(), indicesOptions, metadata,
                         authorizedIndices, replaceWildcards, indicesRequest.includeDataStreams());
-                if (indicesOptions.ignoreUnavailable()) {
-                    //out of all the explicit names (expanded from wildcards and original ones that were left untouched)
-                    //remove all the ones that the current user is not authorized for and ignore them
-                    replaced = replaced.stream().filter(authorizedIndices::contains).collect(Collectors.toList());
-                }
                 resolvedIndicesBuilder.addLocal(replaced);
                 resolvedIndicesBuilder.addRemote(split.getRemote());
             }
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-8b68bf60c98.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-8b68bf60c98.jar.sha1
new file mode 100644
index 0000000000000..6815a521b7287
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-8b68bf60c98.jar.sha1
@@ -0,0 +1 @@
+6f5d5f4968f6e2f066a532d0c7a8ed0fbb9aec7d
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-cfd9f9f98f7.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
deleted file mode 100644
index d4cc15a3c336f..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-9.0.0-snapshot-cfd9f9f98f7.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9c2860726c0ce9a32b6c90976e2338e70c9ebfd2
\ No newline at end of file
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml
index 75dbe35db889c..8f4414396445f 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml
@@ -103,7 +103,6 @@ setup:
   - match:  { jobs.0.state: opened }
   - is_true:  jobs.0.node.name
   - is_true:  jobs.0.node.transport_address
-  - match:  { jobs.0.node.attributes.ml\.max_open_jobs: "512"}
   - is_true:  jobs.0.open_time
   - match:  { jobs.0.timing_stats.job_id: job-stats-test }
   - match:  { jobs.0.timing_stats.bucket_count: 1 }  # Records are 1h apart and bucket span is 1h so 1 bucket is produced
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java
index efdf432bb5d66..e6c0b9cf5bc05 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java
@@ -173,7 +173,7 @@ public void testCreateLatestVersionedIndexIfRequired_GivenShardInitializationPen
         doAnswer(invocationOnMock -> {
             @SuppressWarnings("unchecked")
             ActionListener<ClusterHealthResponse> listener = (ActionListener<ClusterHealthResponse>) invocationOnMock.getArguments()[1];
-            listener.onResponse(new ClusterHealthResponse("", new String[]{}, ClusterState.EMPTY_STATE, false));
+            listener.onResponse(new ClusterHealthResponse("", Strings.EMPTY_ARRAY, ClusterState.EMPTY_STATE, false));
             return null;
         }).when(clusterClient).health(any(), any());
 
@@ -257,7 +257,7 @@ public void testCreateLatestVersionedIndexIfRequired_GivenConcurrentCreationShar
         doAnswer(invocationOnMock -> {
             @SuppressWarnings("unchecked")
             ActionListener<ClusterHealthResponse> listener = (ActionListener<ClusterHealthResponse>) invocationOnMock.getArguments()[1];
-            listener.onResponse(new ClusterHealthResponse("", new String[]{}, ClusterState.EMPTY_STATE, false));
+            listener.onResponse(new ClusterHealthResponse("", Strings.EMPTY_ARRAY, ClusterState.EMPTY_STATE, false));
             return null;
         }).when(clusterClient).health(any(), any());
 
diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle
index 23040f5725edc..4ccad6c71869d 100644
--- a/x-pack/qa/kerberos-tests/build.gradle
+++ b/x-pack/qa/kerberos-tests/build.gradle
@@ -53,7 +53,7 @@ tasks.named("integTest").configure {
   nonInputProperties.systemProperty 'test.userkt', "peppa@${realm}"
   nonInputProperties.systemProperty 'test.userkt.keytab', "${peppaKeytab}"
   nonInputProperties.systemProperty 'test.userpwd', "george@${realm}"
-  systemProperty 'test.userpwd.password', "dino"
+  systemProperty 'test.userpwd.password', "dino_but_longer_than_14_chars"
   systemProperty 'tests.security.manager', 'true'
   jvmArgs([
     "-Djava.security.krb5.conf=${project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("peppa")}",
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml
index 1e7b848190d92..00121f369bbc0 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml
@@ -157,6 +157,6 @@ setup:
       indices.get_mapping:
         index: .ml-annotations-write
 
-  - match: { \.ml-annotations-6.mappings.properties.type.type: "keyword" }
-  - match: { \.ml-annotations-6.mappings.properties.event.type: "keyword" }
-  - match: { \.ml-annotations-6.mappings.properties.detector_index.type: "integer" }
+  - match: { \.ml-annotations-000001.mappings.properties.type.type: "keyword" }
+  - match: { \.ml-annotations-000001.mappings.properties.event.type: "keyword" }
+  - match: { \.ml-annotations-000001.mappings.properties.detector_index.type: "integer" }