Merge branch 'main' into fix_VectorSystemPropertyTests

docs/reference/landing-page.asciidoc

@@ -79,6 +79,11 @@
 
 <h3 class="gtk">Get to know Elasticsearch</h3>
 
+<p>
+<em>Demos:</em>
+  <a href="https://www.elastic.co/demo-gallery?solutions=search&features=null&type=hands-on-learning">Hands-on learning for Search</a>
+</p>
+
 <p>
   <em>New webinar:</em>
   <a href="https://www.elastic.co/virtual-events/architecting-search-apps-on-google-cloud">Architect search apps with Google Cloud</a>

muted-tests.yml

@@ -225,9 +225,6 @@ tests:
 - class: org.elasticsearch.xpack.restart.MLModelDeploymentFullClusterRestartIT
   method: testDeploymentSurvivesRestart {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/115528
-- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
-  method: test {yaml=reference/esql/esql-across-clusters/line_197}
-  issue: https://github.com/elastic/elasticsearch/issues/115575
 - class: org.elasticsearch.oldrepos.OldRepositoryAccessIT
   method: testOldRepoAccess
   issue: https://github.com/elastic/elasticsearch/issues/115631
@@ -284,6 +281,9 @@ tests:
 - class: org.elasticsearch.index.reindex.ReindexNodeShutdownIT
   method: testReindexWithShutdown
   issue: https://github.com/elastic/elasticsearch/issues/115996
+- class: org.elasticsearch.search.query.SearchQueryIT
+  method: testAllDocsQueryString
+  issue: https://github.com/elastic/elasticsearch/issues/115728
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/env/BuildVersion.java

@@ -11,9 +11,13 @@
 
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.internal.BuildExtension;
 import org.elasticsearch.plugins.ExtensionLoader;
+import org.elasticsearch.xcontent.ToXContentFragment;
 
+import java.io.IOException;
 import java.util.ServiceLoader;
 
 /**
@@ -31,7 +35,7 @@
  * provide versions that accommodate different release models or versioning
  * schemes.</p>
  */
-public abstract class BuildVersion {
+public abstract class BuildVersion implements ToXContentFragment, Writeable {
 
     /**
      * Check whether this version is on or after a minimum threshold.
@@ -58,6 +62,11 @@ public abstract class BuildVersion {
      */
     public abstract boolean isFutureVersion();
 
+    /**
+     * Returns this build version in a form suitable for storing in node metadata
+     */
+    public abstract String toNodeMetadata();
+
     /**
      * Create a {@link BuildVersion} from a version ID number.
      *
@@ -72,6 +81,16 @@ public static BuildVersion fromVersionId(int versionId) {
         return CurrentExtensionHolder.BUILD_EXTENSION.fromVersionId(versionId);
     }
 
+    /**
+     * Create a {@link BuildVersion} from a version in node metadata
+     *
+     * @param version The string stored in node metadata
+     * @return a version representing a build or release of Elasticsearch
+     */
+    public static BuildVersion fromNodeMetadata(String version) {
+        return CurrentExtensionHolder.BUILD_EXTENSION.fromNodeMetadata(version);
+    }
+
     /**
      * Create a {@link BuildVersion} from a version string.
      *
@@ -82,6 +101,16 @@ public static BuildVersion fromString(String version) {
         return CurrentExtensionHolder.BUILD_EXTENSION.fromString(version);
     }
 
+    /**
+     * Read a {@link BuildVersion} from an input stream
+     *
+     * @param input The stream to read
+     * @return a version representing a build or release of Elasticsearch
+     */
+    public static BuildVersion fromStream(StreamInput input) throws IOException {
+        return CurrentExtensionHolder.BUILD_EXTENSION.fromStream(input);
+    }
+
     /**
      * Get the current build version.
      *
@@ -94,9 +123,6 @@ public static BuildVersion current() {
         return CurrentExtensionHolder.BUILD_EXTENSION.currentBuildVersion();
     }
 
-    // only exists for NodeMetadata#toXContent
-    public abstract int id();
-
     private static class CurrentExtensionHolder {
         private static final BuildExtension BUILD_EXTENSION = findExtension();
 
@@ -125,6 +151,10 @@ public BuildVersion fromVersionId(int versionId) {
         public BuildVersion fromString(String version) {
             return new DefaultBuildVersion(version);
         }
-    }
 
+        @Override
+        public BuildVersion fromStream(StreamInput in) throws IOException {
+            return new DefaultBuildVersion(in);
+        }
+    }
 }

server/src/main/java/org/elasticsearch/env/DefaultBuildVersion.java

@@ -10,7 +10,11 @@
 package org.elasticsearch.env;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xcontent.XContentBuilder;
 
+import java.io.IOException;
 import java.util.Objects;
 
 /**
@@ -28,7 +32,7 @@ final class DefaultBuildVersion extends BuildVersion {
 
     public static BuildVersion CURRENT = new DefaultBuildVersion(Version.CURRENT.id());
 
-    private final Version version;
+    final Version version;
 
     DefaultBuildVersion(int versionId) {
         assert versionId >= 0 : "Release version IDs must be non-negative integers";
@@ -39,6 +43,10 @@ final class DefaultBuildVersion extends BuildVersion {
         this.version = Version.fromString(Objects.requireNonNull(version));
     }
 
+    DefaultBuildVersion(StreamInput in) throws IOException {
+        this(in.readVInt());
+    }
+
     @Override
     public boolean onOrAfterMinimumCompatible() {
         return Version.CURRENT.minimumCompatibilityVersion().onOrBefore(version);
@@ -50,8 +58,18 @@ public boolean isFutureVersion() {
     }
 
     @Override
-    public int id() {
-        return version.id();
+    public String toNodeMetadata() {
+        return Integer.toString(version.id());
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.value(version.id());
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(version.id());
     }
 
     @Override

server/src/main/java/org/elasticsearch/env/NodeMetadata.java

@@ -204,7 +204,7 @@ protected XContentBuilder newXContentBuilder(XContentType type, OutputStream str
         @Override
         public void toXContent(XContentBuilder builder, NodeMetadata nodeMetadata) throws IOException {
             builder.field(NODE_ID_KEY, nodeMetadata.nodeId);
-            builder.field(NODE_VERSION_KEY, nodeMetadata.nodeVersion.id());
+            builder.field(NODE_VERSION_KEY, nodeMetadata.nodeVersion);
             builder.field(OLDEST_INDEX_VERSION_KEY, nodeMetadata.oldestIndexVersion.id());
         }
 

server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java

@@ -364,7 +364,7 @@ public static NodeMetadata nodeMetadata(Path... dataPaths) throws IOException {
                         );
                     } else if (nodeId == null) {
                         nodeId = thisNodeId;
-                        version = BuildVersion.fromVersionId(Integer.parseInt(userData.get(NODE_VERSION_KEY)));
+                        version = BuildVersion.fromNodeMetadata(userData.get(NODE_VERSION_KEY));
                         if (userData.containsKey(OLDEST_INDEX_VERSION_KEY)) {
                             oldestIndexVersion = IndexVersion.fromId(Integer.parseInt(userData.get(OLDEST_INDEX_VERSION_KEY)));
                         } else {
@@ -395,7 +395,7 @@ public static void overrideVersion(BuildVersion newVersion, Path... dataPaths) t
 
                     try (IndexWriter indexWriter = createIndexWriter(new NIOFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)), true)) {
                         final Map<String, String> commitData = new HashMap<>(userData);
-                        commitData.put(NODE_VERSION_KEY, Integer.toString(newVersion.id()));
+                        commitData.put(NODE_VERSION_KEY, newVersion.toNodeMetadata());
                         commitData.put(OVERRIDDEN_NODE_VERSION_KEY, Boolean.toString(true));
                         indexWriter.setLiveCommitData(commitData.entrySet());
                         indexWriter.commit();
@@ -852,7 +852,7 @@ void prepareCommit(
             final Map<String, String> commitData = Maps.newMapWithExpectedSize(COMMIT_DATA_SIZE);
             commitData.put(CURRENT_TERM_KEY, Long.toString(currentTerm));
             commitData.put(LAST_ACCEPTED_VERSION_KEY, Long.toString(lastAcceptedVersion));
-            commitData.put(NODE_VERSION_KEY, Integer.toString(BuildVersion.current().id()));
+            commitData.put(NODE_VERSION_KEY, BuildVersion.current().toNodeMetadata());
             commitData.put(OLDEST_INDEX_VERSION_KEY, Integer.toString(oldestIndexVersion.id()));
             commitData.put(NODE_ID_KEY, nodeId);
             commitData.put(CLUSTER_UUID_KEY, clusterUUID);

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -242,7 +242,7 @@ public SourceFieldMapper build() {
         }
 
         private Mode resolveSourceMode() {
-            // If the `index.mapper.source.mode` exists it takes precedence to determine the source mode for `_source`
+            // If the `index.mapping.source.mode` exists it takes precedence to determine the source mode for `_source`
             // otherwise the mode is determined according to `_source.mode`.
             if (INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) {
                 return INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings);
@@ -439,6 +439,10 @@ public static boolean isSynthetic(IndexSettings indexSettings) {
         return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC;
     }
 
+    public static boolean isStored(IndexSettings indexSettings) {
+        return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == Mode.STORED;
+    }
+
     public boolean isDisabled() {
         return mode == Mode.DISABLED;
     }

server/src/main/java/org/elasticsearch/internal/BuildExtension.java

@@ -10,8 +10,11 @@
 package org.elasticsearch.internal;
 
 import org.elasticsearch.Build;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.env.BuildVersion;
 
+import java.io.IOException;
+
 /**
  * Allows plugging in current build info.
  */
@@ -39,8 +42,20 @@ default boolean hasReleaseVersioning() {
      */
     BuildVersion fromVersionId(int versionId);
 
+    /**
+     * Returns the {@link BuildVersion} as read from node metadata
+     */
+    default BuildVersion fromNodeMetadata(String version) {
+        return fromVersionId(Integer.parseInt(version));
+    }
+
     /**
      * Returns the {@link BuildVersion} for a given version string.
      */
     BuildVersion fromString(String version);
+
+    /**
+     * Reads a {@link BuildVersion} from the given stream
+     */
+    BuildVersion fromStream(StreamInput in) throws IOException;
 }

server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersion.java

@@ -48,12 +48,12 @@ public static ReservedStateVersion parse(XContentParser parser) {
     }
 
     public static ReservedStateVersion readFrom(StreamInput input) throws IOException {
-        return new ReservedStateVersion(input.readLong(), BuildVersion.fromVersionId(input.readVInt()));
+        return new ReservedStateVersion(input.readLong(), BuildVersion.fromStream(input));
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeLong(version());
-        out.writeVInt(buildVersion().id());
+        out.writeLong(version);
+        buildVersion.writeTo(out);
     }
 }

server/src/test/java/org/elasticsearch/env/BuildVersionTests.java

@@ -42,4 +42,8 @@ public void testIsFutureVersion() {
         assertFalse(afterMinCompat.isFutureVersion());
         assertTrue(futureVersion.isFutureVersion());
     }
+
+    public static BuildVersion increment(BuildVersion version) {
+        return BuildVersion.fromVersionId(((DefaultBuildVersion) version).version.id() + 1);
+    }
 }

server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.env.BuildVersion;
+import org.elasticsearch.env.BuildVersionTests;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
@@ -519,7 +520,7 @@ public void testCheckMetadataVersion() {
 
         task = new ReservedStateUpdateTask(
             "test",
-            new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, BuildVersion.fromVersionId(BuildVersion.current().id() + 1))),
+            new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, BuildVersionTests.increment(BuildVersion.current()))),
             ReservedStateVersionCheck.HIGHER_VERSION_ONLY,
             Map.of(),
             List.of(),
@@ -529,7 +530,7 @@ public void testCheckMetadataVersion() {
         assertThat("Cluster state should not be modified", task.execute(state), sameInstance(state));
         task = new ReservedStateUpdateTask(
             "test",
-            new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, BuildVersion.fromVersionId(BuildVersion.current().id() + 1))),
+            new ReservedStateChunk(Map.of(), new ReservedStateVersion(124L, BuildVersionTests.increment(BuildVersion.current()))),
             ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION,
             Map.of(),
             List.of(),

x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.engine.ReadOnlyEngine;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.TranslogStats;
 import org.elasticsearch.repositories.FilterRepository;
@@ -134,8 +135,9 @@ private static Metadata metadataToSnapshot(Collection<IndexId> indices, Metadata
     @Override
     public void snapshotShard(SnapshotShardContext context) {
         final MapperService mapperService = context.mapperService();
-        if (mapperService.documentMapper() != null // if there is no mapping this is null
-            && mapperService.documentMapper().sourceMapper().isComplete() == false) {
+        if ((mapperService.documentMapper() != null // if there is no mapping this is null
+            && mapperService.documentMapper().sourceMapper().isComplete() == false)
+            || (mapperService.documentMapper() == null && SourceFieldMapper.isStored(mapperService.getIndexSettings()) == false)) {
             context.onFailure(
                 new IllegalStateException(
                     "Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source"