Merge branch 'upstream/master' into reuse-date-stream-timestamp

* upstream/master: (24 commits)
  Implement framework for migrating system indices (elastic#78951)
  Improve transient settings deprecation message (elastic#79504)
  Remove getValue and getValues from Field (elastic#79516)
  Store Template's mappings as bytes for disk serialization (elastic#78746)
  [ML] Add queue_capacity setting to start deployment API (elastic#79433)
  [ML] muting rest compat test issue elastic#79518 (elastic#79519)
  Avoid redundant available indices check (elastic#76540)
  Re-enable BWC tests
  TEST Ensure password 14 chars length on Kerberos FIPS tests (elastic#79496)
  [DOCS] Temporarily remove APM links (elastic#79411)
  Fix CCSDuelIT for skipped shards (elastic#79490)
  Add other time accounting in HotThreads (elastic#79392)
  Add deprecation info API entries for deprecated monitoring settings (elastic#78799)
  Add note in breaking changes for nameid_format (elastic#77785)
  Use 'migration' instead of 'upgrade' in GET system feature migration status responses (elastic#79302)
  Upgrade lucene version 8b68bf60c98 (elastic#79461)
  Use Strings#EMPTY_ARRAY (elastic#79452)
  Quicker shared cache file preallocation (elastic#79447)
  [ML] Removing some code that's obsolete for 8.0 (elastic#79444)
  Ensure indexing_data CCR requests are compressed (elastic#79413)
  ...

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 9.0.0-snapshot-cfd9f9f98f7
+lucene            = 9.0.0-snapshot-8b68bf60c98
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17+35

build.gradle

@@ -132,9 +132,9 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
+boolean bwc_tests_enabled = true
 // place a PR link here when committing bwc changes:
-String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/79385"
+String bwc_tests_disabled_issue = ""
 /*
  * FIPS 140-2 behavior was fixed in 7.11.0. Before that there is no way to run elasticsearch in a
  * JVM that is properly configured to be in fips mode with BCFIPS. For now we need to disable

client/rest-high-level/src/main/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponse.java

@@ -25,7 +25,7 @@
 public class GetFeatureUpgradeStatusResponse {
 
     private static final ParseField FEATURE_UPGRADE_STATUSES = new ParseField("features");
-    private static final ParseField UPGRADE_STATUS = new ParseField("upgrade_status");
+    private static final ParseField UPGRADE_STATUS = new ParseField("migration_status");
 
     private final List<FeatureUpgradeStatus> featureUpgradeStatuses;
     private final String upgradeStatus;
@@ -76,7 +76,7 @@ public static class FeatureUpgradeStatus {
 
         private static final ParseField FEATURE_NAME = new ParseField("feature_name");
         private static final ParseField MINIMUM_INDEX_VERSION = new ParseField("minimum_index_version");
-        private static final ParseField UPGRADE_STATUS = new ParseField("upgrade_status");
+        private static final ParseField UPGRADE_STATUS = new ParseField("migration_status");
         private static final ParseField INDEX_VERSIONS = new ParseField("indices");
 
         @SuppressWarnings("unchecked")

client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java

@@ -23,8 +23,8 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
 
 public class MigrationIT extends ESRestHighLevelClientTestCase {
 
@@ -42,7 +42,7 @@ public void testGetDeprecationInfo() throws IOException {
     public void testGetFeatureUpgradeStatus() throws IOException {
         GetFeatureUpgradeStatusRequest request = new GetFeatureUpgradeStatusRequest();
         GetFeatureUpgradeStatusResponse response = highLevelClient().migration().getFeatureUpgradeStatus(request, RequestOptions.DEFAULT);
-        assertThat(response.getUpgradeStatus(), equalTo("NO_UPGRADE_NEEDED"));
+        assertThat(response.getUpgradeStatus(), equalTo("NO_MIGRATION_NEEDED"));
         assertThat(response.getFeatureUpgradeStatuses().size(), greaterThanOrEqualTo(1));
         Optional<GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus> optionalTasksStatus = response.getFeatureUpgradeStatuses().stream()
             .filter(status -> "tasks".equals(status.getFeatureName()))
@@ -52,19 +52,16 @@ public void testGetFeatureUpgradeStatus() throws IOException {
 
         GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus tasksStatus = optionalTasksStatus.get();
 
-        assertThat(tasksStatus.getUpgradeStatus(), equalTo("NO_UPGRADE_NEEDED"));
+        assertThat(tasksStatus.getUpgradeStatus(), equalTo("NO_MIGRATION_NEEDED"));
         assertThat(tasksStatus.getMinimumIndexVersion(), equalTo(Version.CURRENT.toString()));
         assertThat(tasksStatus.getFeatureName(), equalTo("tasks"));
     }
 
     public void testPostFeatureUpgradeStatus() throws IOException {
         PostFeatureUpgradeRequest request = new PostFeatureUpgradeRequest();
         PostFeatureUpgradeResponse response = highLevelClient().migration().postFeatureUpgrade(request, RequestOptions.DEFAULT);
-        assertThat(response.isAccepted(), equalTo(true));
-        assertThat(response.getFeatures().size(), equalTo(1));
-        PostFeatureUpgradeResponse.Feature feature = response.getFeatures().get(0);
-        assertThat(feature.getFeatureName(), equalTo("security"));
-        assertThat(response.getReason(), nullValue());
-        assertThat(response.getElasticsearchException(), nullValue());
+        assertThat(response.isAccepted(), equalTo(false));
+        assertThat(response.getFeatures(), hasSize(0));
+        assertThat(response.getReason(), equalTo("No system indices require migration"));
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.test.ESTestCase;
 
@@ -79,7 +80,7 @@ private static void toXContent(GetComponentTemplatesResponse response, XContentB
             builder.startObject();
             builder.field("name", e.getKey());
             builder.field("component_template");
-            e.getValue().toXContent(builder, null);
+            e.getValue().toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
         }
         builder.endArray();

client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java

@@ -10,6 +10,7 @@
 
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -53,7 +54,7 @@ private static void toXContent(GetComposableIndexTemplatesResponse response, XCo
             builder.startObject();
             builder.field("name", e.getKey());
             builder.field("index_template");
-            e.getValue().toXContent(builder, null);
+            e.getValue().toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
         }
         builder.endArray();

client/rest-high-level/src/test/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponseTests.java

@@ -40,10 +40,14 @@ protected org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStat
                     randomAlphaOfLengthBetween(3, 20),
                     randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()),
                     randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()),
-                    randomList(4,
-                        () -> new org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexVersion(
+                    randomList(
+                        4,
+                        () -> new org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexInfo(
                             randomAlphaOfLengthBetween(3, 20),
-                            randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion())))
+                            randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()),
+                            null
+                        )
+                    )
                 )),
             randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values())
         );
@@ -78,12 +82,12 @@ protected void assertInstances(
             assertThat(clientStatus.getIndexVersions(), hasSize(serverTestStatus.getIndexVersions().size()));
 
             for (int j = 0; i < clientStatus.getIndexVersions().size(); i++) {
-                org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexVersion serverIndexVersion
+                org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexInfo serverIndexInfo
                     = serverTestStatus.getIndexVersions().get(j);
                 GetFeatureUpgradeStatusResponse.IndexVersion clientIndexVersion = clientStatus.getIndexVersions().get(j);
 
-                assertThat(clientIndexVersion.getIndexName(), equalTo(serverIndexVersion.getIndexName()));
-                assertThat(clientIndexVersion.getVersion(), equalTo(serverIndexVersion.getVersion().toString()));
+                assertThat(clientIndexVersion.getIndexName(), equalTo(serverIndexInfo.getIndexName()));
+                assertThat(clientIndexVersion.getVersion(), equalTo(serverIndexInfo.getVersion().toString()));
             }
         }
     }

docs/reference/migration/apis/feature_upgrade.asciidoc

@@ -24,7 +24,7 @@ and to trigger an automated system upgrade that might potentially involve downti
 ==== {api-prereq-title}
 
 * If the {es} {security-features} are enabled, you must have the `manage`
-<<privileges-list-cluster,cluster privilege>> to use this API. (TODO: true?)
+<<privileges-list-cluster,cluster privilege>> to use this API.
 
 [[feature-upgrade-api-example]]
 ==== {api-examples-title}
@@ -46,79 +46,80 @@ Example response:
     {
       "feature_name" : "async_search",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "enrich",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "fleet",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "geoip",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "kibana",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "logstash_management",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "machine_learning",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "searchable_snapshots",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "security",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "tasks",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "transform",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "watcher",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     }
   ],
-  "upgrade_status" : "NO_UPGRADE_NEEDED"
+  "migration_status" : "NO_MIGRATION_NEEDED"
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"minimum_index_version" : "8.0.0"/"minimum_index_version" : $body.$_path/]
 
 This response tells us that Elasticsearch security needs its internal
 indices upgraded before we can upgrade the cluster to 8.0.
@@ -143,6 +144,7 @@ Example response:
   ]
 }
 --------------------------------------------------
+// TESTRESPONSE[skip: can't actually upgrade system indices in these tests]
 
 This tells us that the security index is being upgraded. To check the
 overall status of the upgrade, call the endpoint with GET.

docs/reference/migration/migrate_8_0.asciidoc

@@ -25,7 +25,6 @@ coming[8.0.0]
 * <<breaking_80_ingest_changes>>
 * <<breaking_80_java_changes>>
 * <<breaking_80_mappings_changes>>
-* <<breaking_80_monitoring_changes>>
 * <<breaking_80_network_changes>>
 * <<breaking_80_node_changes>>
 * <<breaking_80_packaging_changes>>
@@ -128,7 +127,6 @@ include::migrate_8_0/indices.asciidoc[]
 include::migrate_8_0/ingest.asciidoc[]
 include::migrate_8_0/java.asciidoc[]
 include::migrate_8_0/mappings.asciidoc[]
-include::migrate_8_0/monitoring.asciidoc[]
 include::migrate_8_0/network.asciidoc[]
 include::migrate_8_0/node.asciidoc[]
 include::migrate_8_0/packaging.asciidoc[]

docs/reference/migration/migrate_8_0/security.asciidoc

@@ -165,6 +165,37 @@ Specifying this setting in a transport profile in `elasticsearch.yml` will
 result in an error on startup.
 ====
 
+[discrete]
+[[saml-realm-nameid-changes]]
+.The `nameid_format` SAML realm setting no longer has a default value.
+[%collapsible]
+====
+*Details* +
+In SAML, Identity Providers (IdPs) can either be explicitly configured to
+release a `NameID` with a specific format, or configured to attempt to conform 
+with the requirements of a Service Provider (SP). The SP declares its
+requirements in the `NameIDPolicy` element of a SAML Authentication Request.
+In {es}, the `nameid_format` SAML realm setting controls the `NameIDPolicy`
+value.
+
+Previously, the default value for `nameid_format` was
+`urn:oasis:names:tc:SAML:2.0:nameid-format:transient`. This setting created
+authentication requests that required the IdP to release `NameID` with a
+`transient` format.
+
+The default value has been removed, which means that {es} will create SAML Authentication Requests by default that don't put this requirement on the
+IdP. If you want to retain the previous behavior, set `nameid_format` to
+`urn:oasis:names:tc:SAML:2.0:nameid-format:transient`.
+
+*Impact* +
+If you currently don't configure `nameid_format` explicitly, it's possible
+that your IdP will reject authentication requests from {es} because the requests 
+do not specify a `NameID` format (and your IdP is configured to expect one).
+This mismatch can result in a broken SAML configuration. If you're unsure whether 
+your IdP is explicitly configured to use a certain `NameID` format and you want to retain current behavior
+, try setting `nameid_format` to `urn:oasis:names:tc:SAML:2.0:nameid-format:transient` explicitly.
+====
+
 [discrete]
 [[ssl-validation-changes]]
 ===== SSL/TLS configuration validation
@@ -287,7 +318,7 @@ on startup.
 
 [discrete]
 [[ssl-misc-changes]]
-===== Other SSL/TLS changes 
+===== Other SSL/TLS changes
 
 .PKCS#11 keystores and trustores cannot be configured in `elasticsearch.yml`
 [%collapsible]
@@ -307,7 +338,7 @@ Use of a PKCS#11 keystore or truststore as the JRE's default store is not affect
 
 *Impact* +
 If you have a PKCS#11 keystore configured within your `elasticsearch.yml` file, you must remove that
-configuration and switch to a supported keystore type, or configure your PKCS#11 keystore as the 
+configuration and switch to a supported keystore type, or configure your PKCS#11 keystore as the
 JRE default store.
 ====
 
@@ -360,6 +391,7 @@ renamed to better reflect its intended use.
 Assign users with the `kibana_user` role to the `kibana_admin` role.
 Discontinue use of the `kibana_user` role.
 ====
+
 // end::notable-breaking-changes[]
 
 // These are non-notable changes
@@ -373,7 +405,7 @@ Discontinue use of the `kibana_user` role.
 [%collapsible]
 ====
 *Details* +
-If `xpack.security.fips_mode.enabled` is true (see <<fips-140-compliance>>), 
+If `xpack.security.fips_mode.enabled` is true (see <<fips-140-compliance>>),
 the value of `xpack.security.authc.password_hashing.algorithm` now defaults to
 `pbkdf2_stretch`.