Merge branch 'main' into esql-add-sort-to-some-tests

not-napoleon · Dec 13, 2024 · fcf0435 · fcf0435
2 parents 3243d6b + ac0ba24
commit fcf0435
Show file tree

Hide file tree

Showing 51 changed files with 1,527 additions and 316 deletions.
diff --git a/.buildkite/pipelines/pull-request/part-1-entitlements.yml b/.buildkite/pipelines/pull-request/part-1-entitlements.yml
@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-1-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart1
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
diff --git a/.buildkite/pipelines/pull-request/part-2-entitlements.yml b/.buildkite/pipelines/pull-request/part-2-entitlements.yml
@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-2-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart2
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
diff --git a/.buildkite/pipelines/pull-request/part-3-entitlements.yml b/.buildkite/pipelines/pull-request/part-3-entitlements.yml
@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-3-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart3
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
diff --git a/.buildkite/pipelines/pull-request/part-4-entitlements.yml b/.buildkite/pipelines/pull-request/part-4-entitlements.yml
@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-4-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
diff --git a/.buildkite/pipelines/pull-request/part-5-entitlements.yml b/.buildkite/pipelines/pull-request/part-5-entitlements.yml
@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-5-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart5
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
@@ -876,10 +876,8 @@ private void startElasticsearchProcess() {
         // Don't inherit anything from the environment for as that would lack reproducibility
         environment.clear();
         environment.putAll(getESEnvironment());
-        if (cliJvmArgs.isEmpty() == false) {
-            String cliJvmArgsString = String.join(" ", cliJvmArgs);
-            environment.put("CLI_JAVA_OPTS", cliJvmArgsString);
-        }
+        String cliJvmArgsString = String.join(" ", cliJvmArgs);
+        environment.put("CLI_JAVA_OPTS", cliJvmArgsString + " " + System.getProperty("tests.jvm.argline", ""));
 
         // Direct the stderr to the ES log file. This should capture any jvm problems to start.
         // Stdout is discarded because ES duplicates the log file to stdout when run in the foreground.

diff --git a/docs/changelog/116996.yaml b/docs/changelog/116996.yaml
@@ -0,0 +1,5 @@
+pr: 116996
+summary: Initial work on `ReindexDatastreamIndexAction`
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/118410.yaml b/docs/changelog/118410.yaml
@@ -0,0 +1,5 @@
+pr: 118410
+summary: Push down filter passed lookup join
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/118516.yaml b/docs/changelog/118516.yaml
@@ -0,0 +1,6 @@
+pr: 118435
+summary: Fix moving function linear weighted avg
+area: Aggregations
+type: bug
+issues:
+  - 113751
diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle
@@ -48,4 +48,5 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.skipTest("aggregations/date_agg_per_day_of_week/Date aggregartion per day of week", "week-date behaviour has changed")
   task.skipTest("aggregations/time_series/Configure with no synthetic source", "temporary until backport")
   task.skipTest("aggregations/percentiles_hdr_metric/Negative values test", "returned exception has changed")
+  task.skipTest("aggregations/moving_fn/linearWeightedAvg", "math was wrong in previous versions")
 })
diff --git a/...les/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml b/...les/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml
@@ -255,6 +255,17 @@ linearWeightedAvg:
   - skip:
       features: close_to
 
+  - requires:
+      test_runner_features: [capabilities]
+
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_search
+          parameters: [method, path, parameters, capabilities]
+          capabilities: [moving_fn_right_math]
+      reason: "math not fixed yet"
+
   - do:
       search:
         index: no_gaps
@@ -275,11 +286,11 @@ linearWeightedAvg:
   - match: { hits.total.value: 6 }
   - length: { [email protected]: 6 }
   - is_false: [email protected]
-  - close_to: { [email protected]: { value: 0.500, error: 0.0005 } }
-  - close_to: { [email protected]: { value: 1.250, error: 0.0005 } }
-  - close_to: { [email protected]: { value: 1.000, error: 0.0005 } }
-  - close_to: { [email protected]: { value: 2.250, error: 0.0005 } }
-  - close_to: { [email protected]: { value: 3.500, error: 0.0005 } }
+  - close_to: { [email protected]: { value: 1.000, error: 0.0005 } }
+  - close_to: { [email protected]: { value: 1.667, error: 0.0005 } }
+  - close_to: { [email protected]: { value: 1.333, error: 0.0005 } }
+  - close_to: { [email protected]: { value: 3.000, error: 0.0005 } }
+  - close_to: { [email protected]: { value: 4.667, error: 0.0005 } }
 
   - do:
       search:
@@ -301,11 +312,11 @@ linearWeightedAvg:
   - match: { hits.total.value: 6 }
   - length: { [email protected]: 6 }
   - is_false: [email protected]
-  - close_to: { [email protected]: { value: 0.500, error: 0.0005 } }
-  - close_to: { [email protected]: { value: 1.250, error: 0.0005 } }
-  - close_to: { [email protected]: { value: 1.143, error: 0.0005 } }
-  - close_to: { [email protected]: { value: 2.286, error: 0.0005 } }
-  - close_to: { [email protected]: { value: 3.429, error: 0.0005 } }
+  - close_to: { [email protected]: { value: 1.000, error: 0.0005 } }
+  - close_to: { [email protected]: { value: 1.667, error: 0.0005 } }
+  - close_to: { [email protected]: { value: 1.333, error: 0.0005 } }
+  - close_to: { [email protected]: { value: 2.667, error: 0.0005 } }
+  - close_to: { [email protected]: { value: 4.000, error: 0.0005 } }
 
 ---
 ewma:

diff --git a/muted-tests.yml b/muted-tests.yml
@@ -294,18 +294,6 @@ tests:
 - class: org.elasticsearch.docker.test.DockerYmlTestSuiteIT
   method: test {p0=/11_nodes/Test cat nodes output}
   issue: https://github.com/elastic/elasticsearch/issues/118397
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=migrate/20_reindex_status/Test get reindex status with nonexistent task id}
-  issue: https://github.com/elastic/elasticsearch/issues/118401
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=migrate/10_reindex/Test Reindex With Nonexistent Data Stream}
-  issue: https://github.com/elastic/elasticsearch/issues/118274
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=migrate/10_reindex/Test Reindex With Bad Data Stream Name}
-  issue: https://github.com/elastic/elasticsearch/issues/118272
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=migrate/10_reindex/Test Reindex With Unsupported Mode}
-  issue: https://github.com/elastic/elasticsearch/issues/118273
 - class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT
   method: testEveryActionIsEitherOperatorOnlyOrNonOperator
   issue: https://github.com/elastic/elasticsearch/issues/118220

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java
@@ -556,11 +556,7 @@ public void testUpdateSettings() {
     }
 
     public void testSearchQueryThenFetch() throws Exception {
-        interceptTransportActions(
-            SearchTransportService.QUERY_ACTION_NAME,
-            SearchTransportService.FETCH_ID_ACTION_NAME,
-            SearchTransportService.FREE_CONTEXT_ACTION_NAME
-        );
+        interceptTransportActions(SearchTransportService.QUERY_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME);
 
         String[] randomIndicesOrAliases = randomIndicesOrAliases();
         for (int i = 0; i < randomIndicesOrAliases.length; i++) {
@@ -580,16 +576,13 @@ public void testSearchQueryThenFetch() throws Exception {
             SearchTransportService.QUERY_ACTION_NAME,
             SearchTransportService.FETCH_ID_ACTION_NAME
         );
-        // free context messages are not necessarily sent, but if they are, check their indices
-        assertIndicesSubsetOptionalRequests(Arrays.asList(searchRequest.indices()), SearchTransportService.FREE_CONTEXT_ACTION_NAME);
     }
 
     public void testSearchDfsQueryThenFetch() throws Exception {
         interceptTransportActions(
             SearchTransportService.DFS_ACTION_NAME,
             SearchTransportService.QUERY_ID_ACTION_NAME,
-            SearchTransportService.FETCH_ID_ACTION_NAME,
-            SearchTransportService.FREE_CONTEXT_ACTION_NAME
+            SearchTransportService.FETCH_ID_ACTION_NAME
         );
 
         String[] randomIndicesOrAliases = randomIndicesOrAliases();
@@ -611,8 +604,6 @@ public void testSearchDfsQueryThenFetch() throws Exception {
             SearchTransportService.QUERY_ID_ACTION_NAME,
             SearchTransportService.FETCH_ID_ACTION_NAME
         );
-        // free context messages are not necessarily sent, but if they are, check their indices
-        assertIndicesSubsetOptionalRequests(Arrays.asList(searchRequest.indices()), SearchTransportService.FREE_CONTEXT_ACTION_NAME);
     }
 
     private static void assertSameIndices(IndicesRequest originalRequest, String... actions) {

diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -711,7 +711,7 @@ private void raisePhaseFailure(SearchPhaseExecutionException exception) {
                 try {
                     SearchShardTarget searchShardTarget = entry.getSearchShardTarget();
                     Transport.Connection connection = getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());
-                    sendReleaseSearchContext(entry.getContextId(), connection, getOriginalIndices(entry.getShardIndex()));
+                    sendReleaseSearchContext(entry.getContextId(), connection);
                 } catch (Exception inner) {
                     inner.addSuppressed(exception);
                     logger.trace("failed to release context", inner);
@@ -727,10 +727,10 @@ private void raisePhaseFailure(SearchPhaseExecutionException exception) {
       * @see org.elasticsearch.search.fetch.FetchSearchResult#getContextId()
       *
       */
-    void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) {
+    void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection) {
         assert isPartOfPointInTime(contextId) == false : "Must not release point in time context [" + contextId + "]";
         if (connection != null) {
-            searchTransportService.sendFreeContext(connection, contextId, originalIndices);
+            searchTransportService.sendFreeContext(connection, contextId, ActionListener.noop());
         }
     }
 

diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
@@ -119,11 +119,7 @@ public void onFailure(Exception exception) {
                                 // the query might not have been executed at all (for example because thread pool rejected
                                 // execution) and the search context that was created in dfs phase might not be released.
                                 // release it again to be in the safe side
-                                context.sendReleaseSearchContext(
-                                    querySearchRequest.contextId(),
-                                    connection,
-                                    context.getOriginalIndices(shardIndex)
-                                );
+                                context.sendReleaseSearchContext(querySearchRequest.contextId(), connection);
                             }
                         }
                     }

diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java
@@ -97,11 +97,7 @@ protected static void releaseIrrelevantSearchContext(SearchPhaseResult searchPha
                 context.getLogger().trace("trying to release search context [{}]", phaseResult.getContextId());
                 SearchShardTarget shardTarget = phaseResult.getSearchShardTarget();
                 Transport.Connection connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId());
-                context.sendReleaseSearchContext(
-                    phaseResult.getContextId(),
-                    connection,
-                    context.getOriginalIndices(phaseResult.getShardIndex())
-                );
+                context.sendReleaseSearchContext(phaseResult.getContextId(), connection);
             } catch (Exception e) {
                 context.getLogger().trace("failed to release context", e);
             }

diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -13,12 +13,10 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
-import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.get.TransportGetTaskAction;
 import org.elasticsearch.action.support.ChannelActionListener;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -124,24 +122,6 @@ public SearchTransportService(
         this.responseWrapper = responseWrapper;
     }
 
-    private static final ActionListenerResponseHandler<SearchFreeContextResponse> SEND_FREE_CONTEXT_LISTENER =
-        new ActionListenerResponseHandler<>(
-            ActionListener.noop(),
-            SearchFreeContextResponse::readFrom,
-            TransportResponseHandler.TRANSPORT_WORKER
-        );
-
-    public void sendFreeContext(Transport.Connection connection, final ShardSearchContextId contextId, OriginalIndices originalIndices) {
-        transportService.sendRequest(
-            connection,
-            FREE_CONTEXT_ACTION_NAME,
-            new SearchFreeContextRequest(originalIndices, contextId),
-            TransportRequestOptions.EMPTY,
-            // no need to respond if it was freed or not
-            SEND_FREE_CONTEXT_LISTENER
-        );
-    }
-
     public void sendFreeContext(
         Transport.Connection connection,
         ShardSearchContextId contextId,
@@ -370,43 +350,6 @@ private static class ClearScrollContextsRequest extends TransportRequest {
         }
     }
 
-    static class SearchFreeContextRequest extends ScrollFreeContextRequest implements IndicesRequest {
-        private final OriginalIndices originalIndices;
-
-        SearchFreeContextRequest(OriginalIndices originalIndices, ShardSearchContextId id) {
-            super(id);
-            this.originalIndices = originalIndices;
-        }
-
-        SearchFreeContextRequest(StreamInput in) throws IOException {
-            super(in);
-            originalIndices = OriginalIndices.readOriginalIndices(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            OriginalIndices.writeOriginalIndices(originalIndices, out);
-        }
-
-        @Override
-        public String[] indices() {
-            if (originalIndices == null) {
-                return null;
-            }
-            return originalIndices.indices();
-        }
-
-        @Override
-        public IndicesOptions indicesOptions() {
-            if (originalIndices == null) {
-                return null;
-            }
-            return originalIndices.indicesOptions();
-        }
-
-    }
-
     public static class SearchFreeContextResponse extends TransportResponse {
 
         private static final SearchFreeContextResponse FREED = new SearchFreeContextResponse(true);
@@ -456,12 +399,13 @@ public static void registerRequestHandler(TransportService transportService, Sea
             SearchFreeContextResponse::readFrom
         );
 
-        transportService.registerRequestHandler(
-            FREE_CONTEXT_ACTION_NAME,
-            freeContextExecutor,
-            SearchFreeContextRequest::new,
-            freeContextHandler
-        );
+        // TODO: remove this handler once the lowest compatible version stops using it
+        transportService.registerRequestHandler(FREE_CONTEXT_ACTION_NAME, freeContextExecutor, in -> {
+            var res = new ScrollFreeContextRequest(in);
+            // this handler exists for BwC purposes only, we don't need the original indices to free the context
+            OriginalIndices.readOriginalIndices(in);
+            return res;
+        }, freeContextHandler);
         TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, false, SearchFreeContextResponse::readFrom);
 
         transportService.registerRequestHandler(