Merge branch 'main' into blockFactory

ChrisHegarty · Sep 21, 2023 · e4c5052 · e4c5052
2 parents 7d67a14 + d0e91c9
commit e4c5052
Show file tree

Hide file tree

Showing 256 changed files with 3,598 additions and 1,209 deletions.
diff --git a/...s-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java b/...s-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
@@ -67,6 +67,7 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
         map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-http");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-disabled");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies");

diff --git a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml
@@ -32,7 +32,7 @@
 
   <!-- Intentionally have multi line string for a bulk request, otherwise this needs to fallback to string concatenation  -->
   <suppress files="modules[/\\]data-streams[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]datastreams[/\\]TsdbDataStreamRestIT.java" checks="LineLength" />
-  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
+  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
 
   <!-- Gradle requires inputs to be seriablizable -->
   <suppress files="build-tools-internal[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]internal[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />

diff --git a/docs/changelog/99091.yaml b/docs/changelog/99091.yaml
@@ -0,0 +1,5 @@
+pr: 99091
+summary: Add flamegraph API
+area: Application
+type: enhancement
+issues: []
diff --git a/docs/changelog/99193.yaml b/docs/changelog/99193.yaml
@@ -0,0 +1,5 @@
+pr: 99193
+summary: Wait for cluster state in recovery
+area: Recovery
+type: enhancement
+issues: []
diff --git a/docs/changelog/99278.yaml b/docs/changelog/99278.yaml
@@ -0,0 +1,5 @@
+pr: 99278
+summary: Support rotatating the JWT shared secret
+area: Security
+type: enhancement
+issues: []
diff --git a/docs/changelog/99644.yaml b/docs/changelog/99644.yaml
@@ -0,0 +1,6 @@
+pr: 99644
+summary: Add links to docs from failing bootstrap checks
+area: Infra/Node Lifecycle
+type: enhancement
+issues: [99614]
+
diff --git a/docs/changelog/99682.yaml b/docs/changelog/99682.yaml
@@ -0,0 +1,5 @@
+pr: 99682
+summary: Increase the max vector dims to 4096
+area: Vector Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/99685.yaml b/docs/changelog/99685.yaml
@@ -0,0 +1,5 @@
+pr: 99685
+summary: Fix `advanceExact` for doc values from sources
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/99695.yaml b/docs/changelog/99695.yaml
@@ -0,0 +1,5 @@
+pr: 99695
+summary: "ESQL: Better management of not stored TEXT fiels with synthetic source"
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/99717.yaml b/docs/changelog/99717.yaml
@@ -0,0 +1,5 @@
+pr: 99717
+summary: Treating watcher webhook response header names as case-insensitive
+area: Watcher
+type: bug
+issues: []
diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc
@@ -62,7 +62,7 @@ In many cases, a brute-force kNN search is not efficient enough. For this
 reason, the `dense_vector` type supports indexing vectors into a specialized
 data structure to support fast kNN retrieval through the <<search-api-knn, `knn` option>> in the search API
 
-Unmapped array fields of float elements with size between 128 and 2048 are dynamically mapped as `dense_vector` with a default similariy of `cosine`.
+Unmapped array fields of float elements with size between 128 and 4096 are dynamically mapped as `dense_vector` with a default similariy of `cosine`.
 You can override the default similarity by explicitly mapping the field as `dense_vector` with the desired similarity.
 
 Indexing is enabled by default for dense vector fields.
@@ -132,7 +132,7 @@ integer values between -128 to 127, inclusive for both indexing and searching.
 
 `dims`::
 (Optional, integer)
-Number of vector dimensions. Can't exceed `2048`. If `dims` is not specified,
+Number of vector dimensions. Can't exceed `4096`. If `dims` is not specified,
 it will be set to the length of the first vector added to the field.
 
 `index`::

diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
@@ -7,6 +7,7 @@
 This section summarizes the changes in each release.
 
 * <<release-notes-8.11.0>>
+* <<release-notes-8.10.2>>
 * <<release-notes-8.10.1>>
 * <<release-notes-8.10.0>>
 * <<release-notes-8.9.2>>
@@ -51,6 +52,7 @@ This section summarizes the changes in each release.
 --
 
 include::release-notes/8.11.0.asciidoc[]
+include::release-notes/8.10.2.asciidoc[]
 include::release-notes/8.10.1.asciidoc[]
 include::release-notes/8.10.0.asciidoc[]
 include::release-notes/8.9.2.asciidoc[]

diff --git a/docs/reference/release-notes/8.10.2.asciidoc b/docs/reference/release-notes/8.10.2.asciidoc
@@ -0,0 +1,6 @@
+[[release-notes-8.10.2]]
+== {es} version 8.10.2
+
+8.10.2 contains no significant changes.
+
+Also see <<breaking-changes-8.10,Breaking changes in 8.10>>.
diff --git a/docs/reference/settings/security-settings.asciidoc b/docs/reference/settings/security-settings.asciidoc
@@ -2266,11 +2266,20 @@ restricts which ones are allowed to submit those JWTs to {es}.
 
 // tag::jwt-client-authentication-shared-secret-tag[]
 `client_authentication.shared_secret` {ess-icon}::
-(<<secure-settings,Secure>>)
+(<<secure-settings,Secure>>, <<reloadable-secure-settings,reloadable>>)
 Secret value string for client authentication.
 Required if `client_authentication.type` is `shared_secret`.
 // end::jwt-client-authentication-shared-secret-tag[]
 
+// tag::jwt-client-authentication-rotation-grace-period-tag[]
+`client_authentication.rotation_grace_period`::
+(<<static-cluster-setting,Static>>)
+Sets the grace period for how long after rotating the `client_authentication.shared_secret`
+is valid. `client_authentication.shared_secret` can be rotated by updating the
+keystore then calling the <<cluster-nodes-reload-secure-settings, reload API>>.
+Defaults to `1m`.
+// end::jwt-client-authentication-rotation-grace-period-tag[]
+
 // tag::jwt-http-connect-timeout-tag[]
 `http.connect_timeout` {ess-icon}::
 (<<static-cluster-setting,Static>>)

diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -221,14 +221,6 @@ releases are not suitable for production. The early-access check detects these
 early-access snapshots. To pass this check, you must start Elasticsearch on a
 release build of the JVM.
 
-=== G1GC check
-
-Early versions of the HotSpot JVM that shipped with JDK 8 are known to
-have issues that can lead to index corruption when the G1GC collector is
-enabled. The versions impacted are those earlier than the version of
-HotSpot that shipped with JDK 8u40. The G1GC check detects these early
-versions of the HotSpot JVM.
-
 === All permission check
 
 The all permission check ensures that the security policy used during bootstrap

diff --git a/...analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/...analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
@@ -131,8 +131,8 @@
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.synonyms.SynonymsManagementAPIService;
+import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.tracing.Tracer;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.tartarus.snowball.ext.DutchStemmer;

diff --git a/...mmon/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java b/...mmon/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java
@@ -28,10 +28,10 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.tracing.Tracer;
 
 import java.io.IOException;
 import java.util.Collections;

diff --git a/...on/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java b/...on/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java
@@ -28,10 +28,10 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.tracing.Tracer;
 
 import java.util.Collections;
 

diff --git a/modules/apm/build.gradle b/modules/apm/build.gradle
@@ -9,7 +9,7 @@ apply plugin: 'elasticsearch.internal-es-plugin'
 esplugin {
   name 'apm'
   description 'Provides APM integration for Elasticsearch'
-  classname 'org.elasticsearch.tracing.apm.APM'
+  classname 'org.elasticsearch.telemetry.apm.APM'
 }
 
 def otelVersion = '1.17.0'

diff --git a/modules/apm/src/main/java/module-info.java b/modules/apm/src/main/java/module-info.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-module org.elasticsearch.tracing.apm {
+module org.elasticsearch.telemetry.apm {
     requires org.elasticsearch.base;
     requires org.elasticsearch.server;
     requires org.elasticsearch.xcontent;
@@ -15,5 +15,5 @@
     requires io.opentelemetry.context;
     requires io.opentelemetry.api;
 
-    exports org.elasticsearch.tracing.apm;
+    exports org.elasticsearch.telemetry.apm;
 }
diff --git a/...va/org/elasticsearch/tracing/apm/APM.java → .../org/elasticsearch/telemetry/apm/APM.java b/...va/org/elasticsearch/tracing/apm/APM.java → .../org/elasticsearch/telemetry/apm/APM.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.tracing.apm;
+package org.elasticsearch.telemetry.apm;
 
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.client.internal.Client;
@@ -24,8 +24,10 @@
 import org.elasticsearch.plugins.TracerPlugin;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.telemetry.apm.settings.APMAgentSettings;
+import org.elasticsearch.telemetry.apm.tracing.APMTracer;
+import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.tracing.Tracer;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
@@ -35,7 +37,7 @@
 
 /**
  * This module integrates Elastic's APM product with Elasticsearch. Elasticsearch has
- * a {@link org.elasticsearch.tracing.Tracer} interface, which this module implements via
+ * a {@link org.elasticsearch.telemetry.tracing.Tracer} interface, which this module implements via
  * {@link APMTracer}. We use the OpenTelemetry API to capture "spans", and attach the
  * Elastic APM Java to ship those spans to an APM server. Although it is possible to
  * programmatically attach the agent, the Security Manager permissions required for this

diff --git a/...csearch/tracing/apm/APMAgentSettings.java → ...emetry/apm/settings/APMAgentSettings.java b/...csearch/tracing/apm/APMAgentSettings.java → ...emetry/apm/settings/APMAgentSettings.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.tracing.apm;
+package org.elasticsearch.telemetry.apm.settings;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.telemetry.apm.tracing.APMTracer;
 
 import java.security.AccessController;
 import java.security.PrivilegedAction;
@@ -31,7 +32,7 @@
  * This class is responsible for APM settings, both for Elasticsearch and the APM Java agent.
  * The methods could all be static, however they are not in order to make unit testing easier.
  */
-class APMAgentSettings {
+public class APMAgentSettings {
 
     private static final Logger LOGGER = LogManager.getLogger(APMAgentSettings.class);
 
@@ -41,7 +42,7 @@ class APMAgentSettings {
      */
     static Map<String, String> APM_AGENT_DEFAULT_SETTINGS = Map.of("transaction_sample_rate", "0.2");
 
-    void addClusterSettingsListeners(ClusterService clusterService, APMTracer apmTracer) {
+    public void addClusterSettingsListeners(ClusterService clusterService, APMTracer apmTracer) {
         final ClusterSettings clusterSettings = clusterService.getClusterSettings();
         clusterSettings.addSettingsUpdateConsumer(APM_ENABLED_SETTING, enabled -> {
             apmTracer.setEnabled(enabled);
@@ -59,7 +60,7 @@ void addClusterSettingsListeners(ClusterService clusterService, APMTracer apmTra
      * Copies APM settings from the provided settings object into the corresponding system properties.
      * @param settings the settings to apply
      */
-    void syncAgentSystemProperties(Settings settings) {
+    public void syncAgentSystemProperties(Settings settings) {
         this.setAgentSetting("recording", Boolean.toString(APM_ENABLED_SETTING.get(settings)));
 
         // Apply default values for some system properties. Although we configure
@@ -81,7 +82,7 @@ void syncAgentSystemProperties(Settings settings) {
      * @param value the value to set, or <code>null</code>
      */
     @SuppressForbidden(reason = "Need to be able to manipulate APM agent-related properties to set them dynamically")
-    void setAgentSetting(String key, String value) {
+    public void setAgentSetting(String key, String value) {
         final String completeKey = "elastic.apm." + Objects.requireNonNull(key);
         AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
             if (value == null || value.isEmpty()) {
@@ -107,7 +108,7 @@ void setAgentSetting(String key, String value) {
         "recording"
     );
 
-    static final Setting.AffixSetting<String> APM_AGENT_SETTINGS = Setting.prefixKeySetting(
+    public static final Setting.AffixSetting<String> APM_AGENT_SETTINGS = Setting.prefixKeySetting(
         APM_SETTING_PREFIX + "agent.",
         (qualifiedKey) -> {
             final String[] parts = qualifiedKey.split("\\.");
@@ -122,19 +123,19 @@ void setAgentSetting(String key, String value) {
         }
     );
 
-    static final Setting<List<String>> APM_TRACING_NAMES_INCLUDE_SETTING = Setting.stringListSetting(
+    public static final Setting<List<String>> APM_TRACING_NAMES_INCLUDE_SETTING = Setting.stringListSetting(
         APM_SETTING_PREFIX + "names.include",
         OperatorDynamic,
         NodeScope
     );
 
-    static final Setting<List<String>> APM_TRACING_NAMES_EXCLUDE_SETTING = Setting.stringListSetting(
+    public static final Setting<List<String>> APM_TRACING_NAMES_EXCLUDE_SETTING = Setting.stringListSetting(
         APM_SETTING_PREFIX + "names.exclude",
         OperatorDynamic,
         NodeScope
     );
 
-    static final Setting<List<String>> APM_TRACING_SANITIZE_FIELD_NAMES = Setting.stringListSetting(
+    public static final Setting<List<String>> APM_TRACING_SANITIZE_FIELD_NAMES = Setting.stringListSetting(
         APM_SETTING_PREFIX + "sanitize_field_names",
         List.of(
             "password",
@@ -154,14 +155,17 @@ void setAgentSetting(String key, String value) {
         NodeScope
     );
 
-    static final Setting<Boolean> APM_ENABLED_SETTING = Setting.boolSetting(
+    public static final Setting<Boolean> APM_ENABLED_SETTING = Setting.boolSetting(
         APM_SETTING_PREFIX + "enabled",
         false,
         OperatorDynamic,
         NodeScope
     );
 
-    static final Setting<SecureString> APM_SECRET_TOKEN_SETTING = SecureSetting.secureString(APM_SETTING_PREFIX + "secret_token", null);
+    public static final Setting<SecureString> APM_SECRET_TOKEN_SETTING = SecureSetting.secureString(
+        APM_SETTING_PREFIX + "secret_token",
+        null
+    );
 
-    static final Setting<SecureString> APM_API_KEY_SETTING = SecureSetting.secureString(APM_SETTING_PREFIX + "api_key", null);
+    public static final Setting<SecureString> APM_API_KEY_SETTING = SecureSetting.secureString(APM_SETTING_PREFIX + "api_key", null);
 }