From 8f2b5f10c734526959d9cba0502b1ff14bb9f8bf Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 7 Sep 2023 17:11:19 +0100 Subject: [PATCH 01/69] Fix some more stray references to Version (#99298) --- .../org/elasticsearch/cluster/metadata/IndexMetadata.java | 2 +- .../java/org/elasticsearch/snapshots/RestoreService.java | 7 +++---- .../java/org/elasticsearch/snapshots/SnapshotsService.java | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 5ab39cb265daa..517bc2293ab6b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -2648,7 +2648,7 @@ private static void handleLegacyMapping(Builder builder, Map map } /** - * Return the {@link Version} of Elasticsearch that has been used to create an index given its settings. + * Return the {@link IndexVersion} of Elasticsearch that has been used to create an index given its settings. * * @throws IllegalArgumentException if the given index settings doesn't contain a value for the key * {@value IndexMetadata#SETTING_VERSION_CREATED} diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 335d69c83a4f5..589163ab00581 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.elasticsearch.action.support.IndicesOptions; @@ -970,10 +969,10 @@ static void validateSnapshotRestorable( if (IndexVersion.current().before(snapshotInfo.version())) { throw new SnapshotRestoreException( new Snapshot(repository.name(), snapshotInfo.snapshotId()), - "the snapshot was created with Elasticsearch version [" + "the snapshot was created with index version [" + snapshotInfo.version() - + "] which is higher than the version of this node [" - + Version.CURRENT + + "] which is higher than the version used by this node [" + + IndexVersion.current() + "]" ); } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index f119462e9428d..5fc4353a68230 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -14,7 +14,6 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionRunnable; @@ -2116,8 +2115,9 @@ private void addDeleteListener(String deleteUUID, ActionListener listener) } /** - * Determines the minimum {@link Version} that the snapshot repository must be compatible with from the current nodes in the cluster - * and the contents of the repository. The minimum version is determined as the lowest version found across all snapshots in the + * Determines the minimum {@link IndexVersion} that the snapshot repository must be compatible with + * from the current nodes in the cluster and the contents of the repository. + * The minimum version is determined as the lowest version found across all snapshots in the * repository and all nodes in the cluster. * * @param minNodeVersion minimum node version in the cluster From 87195052d7fe6c37e272e985b2dfc013d6e1a5a9 Mon Sep 17 00:00:00 2001 From: James Rodewig Date: Thu, 7 Sep 2023 12:19:25 -0400 Subject: [PATCH 02/69] [DOCS] Fix syntax errors in Docker docs (#99313) **Problem:** - The `elasticsearch-reset-password` commands in the ES Docker install docs are missing the required `-u` flag - The `ifeval` blocks in the Kibana section of the ES Docker install docs aren't rendering correctly in released docs. **Solution:** - Add the `-u` flag to `elasticsearch-reset-password` examples - Fix the Asciidoc syntax to correctly render the `ifeval` blocks. Example: Rel: https://github.com/elastic/elasticsearch/pull/99112 --- docs/reference/setup/install/docker.asciidoc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc index 6120a7a5391c4..d4ebacb292224 100644 --- a/docs/reference/setup/install/docker.asciidoc +++ b/docs/reference/setup/install/docker.asciidoc @@ -133,7 +133,7 @@ credentials using the following commands. -- [source,sh,subs="attributes"] ---- -docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-reset-password +docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana ---- @@ -202,28 +202,32 @@ curl --cacert http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200/_c . Pull the {kib} Docker image. + +-- ifeval::["{release-state}"=="unreleased"] WARNING: Version {version} of {kib} has not yet been released, so no Docker image is currently available for this version. endif::[] -+ + [source,sh,subs="attributes"] ---- docker pull {kib-docker-image} ---- +-- . Optional: Verify the {kib} image's signature. + +-- ifeval::["{release-state}"=="unreleased"] WARNING: Version {version} of {kib} has not yet been released, so no Docker image signature is currently available for this version. endif::[] -+ + [source,sh,subs="attributes"] ---- wget https://artifacts.elastic.co/cosign.pub cosign verify --key cosign.pub {kib-docker-image} ---- +-- . Start a {kib} container. + @@ -251,7 +255,7 @@ To regenerate the password, run: + [source,sh] ---- -docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-reset-password +docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic ---- [[remove-containers-docker]] From 2461cb152369d6ef92ce0203d97f70596a1e4ecb Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 7 Sep 2023 18:24:25 +0200 Subject: [PATCH 03/69] Remove TransportOptions from ActionType (#97721) This wasn't actually used on this class, inlined the single use case. This sets up a follow-up with a massive code saving, there really is no need to subclass this class 400+ times :) I'll open a PR for that once this is in, it's an easy automatic cleanup. --- .../main/java/org/elasticsearch/action/ActionType.java | 8 -------- .../java/org/elasticsearch/action/bulk/BulkAction.java | 4 ---- .../action/bulk/TransportShardBulkAction.java | 7 ++++++- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/ActionType.java b/server/src/main/java/org/elasticsearch/action/ActionType.java index 4e5e1d9b821d3..d24abf049ec1e 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionType.java +++ b/server/src/main/java/org/elasticsearch/action/ActionType.java @@ -9,7 +9,6 @@ package org.elasticsearch.action; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.transport.TransportRequestOptions; /** * A generic action. Should strive to make it a singleton. @@ -42,13 +41,6 @@ public Writeable.Reader getResponseReader() { return responseReader; } - /** - * Optional request options for the action. - */ - public TransportRequestOptions transportOptions() { - return TransportRequestOptions.EMPTY; - } - @Override public boolean equals(Object o) { return o instanceof ActionType && name.equals(((ActionType) o).name()); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkAction.java index 94f0b94d554b0..70b010b6ca88d 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkAction.java @@ -25,8 +25,4 @@ private BulkAction() { super(NAME, BulkResponse::new); } - @Override - public TransportRequestOptions transportOptions() { - return TRANSPORT_REQUEST_OPTIONS; - } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 27dae7f6272a9..9266ee3ee0b68 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -117,9 +117,14 @@ public TransportShardBulkAction( this.postWriteAction = WriteAckDelay.create(settings, threadPool); } + private static final TransportRequestOptions TRANSPORT_REQUEST_OPTIONS = TransportRequestOptions.of( + null, + TransportRequestOptions.Type.BULK + ); + @Override protected TransportRequestOptions transportOptions() { - return BulkAction.INSTANCE.transportOptions(); + return TRANSPORT_REQUEST_OPTIONS; } @Override From fc2b398cb407d6d56eaf521adae2bd84f5980876 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 7 Sep 2023 19:04:38 +0200 Subject: [PATCH 04/69] Prune changelogs after 7.17.13 release --- docs/changelog/98260.yaml | 5 ----- docs/changelog/98765.yaml | 6 ------ 2 files changed, 11 deletions(-) delete mode 100644 docs/changelog/98260.yaml delete mode 100644 docs/changelog/98765.yaml diff --git a/docs/changelog/98260.yaml b/docs/changelog/98260.yaml deleted file mode 100644 index b33c5cb3ff707..0000000000000 --- a/docs/changelog/98260.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 98260 -summary: Avoid lifecycle NPE in the data stream lifecycle usage API -area: Data streams -type: bug -issues: [] diff --git a/docs/changelog/98765.yaml b/docs/changelog/98765.yaml deleted file mode 100644 index 8d30dd03cd91d..0000000000000 --- a/docs/changelog/98765.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 98765 -summary: Fix mvt error when returning partial results -area: Geo -type: bug -issues: - - 98730 From 2fccd019d5afe2b731997021768c31376611678c Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 7 Sep 2023 19:07:08 +0200 Subject: [PATCH 05/69] Bump versions after 8.9.2 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 16 ++++++++++++++++ .buildkite/pipelines/periodic.yml | 10 ++++++++++ .ci/bwcVersions | 1 + .ci/snapshotBwcVersions | 2 +- .../src/main/java/org/elasticsearch/Version.java | 1 + 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index bd5c8efcf7eda..bdff41788b563 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -40,7 +40,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.14", "8.9.2", "8.10.0", "8.11.0"] + BWC_VERSION: ["7.17.14", "8.9.3", "8.10.0", "8.11.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 4cc7bb92b78ec..3b70b82746ed8 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -1584,6 +1584,22 @@ steps: env: BWC_VERSION: 8.9.2 + - label: "{{matrix.image}} / 8.9.3 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.9.3 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.9.3 + - label: "{{matrix.image}} / 8.10.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.0 timeout_in_minutes: 300 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 6c6012f50d1f6..da72f059acded 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -972,6 +972,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 8.9.2 + - label: 8.9.3 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.9.3#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.9.3 - label: 8.10.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest timeout_in_minutes: 300 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 537e628414f25..42b124a8a0aca 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -96,5 +96,6 @@ BWC_VERSION: - "8.9.0" - "8.9.1" - "8.9.2" + - "8.9.3" - "8.10.0" - "8.11.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 6af86865ed798..da5aa1cdd64e1 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,5 @@ BWC_VERSION: - "7.17.14" - - "8.9.2" + - "8.9.3" - "8.10.0" - "8.11.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index f4d712fd9dedf..37b382db2f2ca 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -147,6 +147,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_9_0 = new Version(8_09_00_99); public static final Version V_8_9_1 = new Version(8_09_01_99); public static final Version V_8_9_2 = new Version(8_09_02_99); + public static final Version V_8_9_3 = new Version(8_09_03_99); public static final Version V_8_10_0 = new Version(8_10_00_99); public static final Version V_8_11_0 = new Version(8_11_00_99); public static final Version CURRENT = V_8_11_0; From db11f470b29040551e69654ca910e7143edf7efc Mon Sep 17 00:00:00 2001 From: Pooya Salehi Date: Thu, 7 Sep 2023 19:14:37 +0200 Subject: [PATCH 06/69] Change GetFromTranslog to indices action (#99300) Move GetFromTranslog to `indices` rather than `internal` to ensure FLS/DLS rules are enforced, and make its request implement `IndicesRequest`. Relates ES-6626 --- docs/changelog/99300.yaml | 5 +++++ .../get/TransportGetFromTranslogAction.java | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/99300.yaml diff --git a/docs/changelog/99300.yaml b/docs/changelog/99300.yaml new file mode 100644 index 0000000000000..508001b98f29e --- /dev/null +++ b/docs/changelog/99300.yaml @@ -0,0 +1,5 @@ +pr: 99300 +summary: Change `GetFromTranslog` to indices action +area: CRUD +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java index c3127c9a2a05d..37e07b3ca2e07 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java @@ -14,8 +14,10 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -41,7 +43,7 @@ public class TransportGetFromTranslogAction extends HandledTransportAction< TransportGetFromTranslogAction.Request, TransportGetFromTranslogAction.Response> { - public static final String NAME = "internal:data/read/get_from_translog"; + public static final String NAME = "indices:data/read/get_from_translog"; public static final Logger logger = LogManager.getLogger(TransportGetFromTranslogAction.class); private final IndicesService indicesService; @@ -83,7 +85,7 @@ protected void doExecute(Task task, Request request, ActionListener li }); } - public static class Request extends ActionRequest { + public static class Request extends ActionRequest implements IndicesRequest { private final GetRequest getRequest; private final ShardId shardId; @@ -123,6 +125,16 @@ public ActionRequestValidationException validate() { public String toString() { return "GetFromTranslogRequest{" + "getRequest=" + getRequest + ", shardId=" + shardId + "}"; } + + @Override + public String[] indices() { + return getRequest.indices(); + } + + @Override + public IndicesOptions indicesOptions() { + return getRequest.indicesOptions(); + } } public static class Response extends ActionResponse { From e2e7b8bc693bbbf198a58eee580e3b7dfd2cef5e Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 7 Sep 2023 19:03:22 +0100 Subject: [PATCH 07/69] Migrate RetryableAction away from name-based executor (#99301) Replaces the executor name with a proper executor, and also removes the constructor overrides that dangerously default to running the retries on the scheduler thread so that callers have to make a conscious choice of executor. Relates #99027 and friends --- .../action/support/RetryableAction.java | 30 ++++--------------- .../replication/ReplicationOperation.java | 4 ++- .../recovery/RemoteRecoveryTargetHandler.java | 10 ++++++- .../action/support/RetryableActionTests.java | 22 +++++++++----- .../PendingReplicationActionsTests.java | 10 ++++++- .../task/OpenJobPersistentTasksExecutor.java | 2 +- .../persistence/ResultsPersisterService.java | 2 +- 7 files changed, 43 insertions(+), 37 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/RetryableAction.java b/server/src/main/java/org/elasticsearch/action/support/RetryableAction.java index 908d08b70a800..3e5114ddcfa9a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/RetryableAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/RetryableAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayDeque; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.core.Strings.format; @@ -39,42 +40,21 @@ public abstract class RetryableAction { private final long timeoutMillis; private final long startMillis; private final ActionListener finalListener; - private final String executor; + private final Executor executor; private volatile Scheduler.ScheduledCancellable retryTask; - public RetryableAction( - Logger logger, - ThreadPool threadPool, - TimeValue initialDelay, - TimeValue timeoutValue, - ActionListener listener - ) { - this(logger, threadPool, initialDelay, timeoutValue, listener, ThreadPool.Names.SAME); - } - public RetryableAction( Logger logger, ThreadPool threadPool, TimeValue initialDelay, TimeValue timeoutValue, ActionListener listener, - String executor + Executor executor ) { this(logger, threadPool, initialDelay, TimeValue.MAX_VALUE, timeoutValue, listener, executor); } - public RetryableAction( - Logger logger, - ThreadPool threadPool, - TimeValue initialDelay, - TimeValue maxDelayBound, - TimeValue timeoutValue, - ActionListener listener - ) { - this(logger, threadPool, initialDelay, maxDelayBound, timeoutValue, listener, ThreadPool.Names.SAME); - } - public RetryableAction( Logger logger, ThreadPool threadPool, @@ -82,7 +62,7 @@ public RetryableAction( TimeValue maxDelayBound, TimeValue timeoutValue, ActionListener listener, - String executor + Executor executor ) { this.logger = logger; this.threadPool = threadPool; @@ -106,7 +86,7 @@ public RetryableAction( public void run() { final RetryingListener retryingListener = new RetryingListener(initialDelayMillis, null); final Runnable runnable = createRunnable(retryingListener); - threadPool.executor(executor).execute(runnable); + executor.execute(runnable); } public void cancel(Exception e) { diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java index 17c2a68f21332..0b4d90a896c10 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -293,7 +294,8 @@ public String toString() { threadPool, initialRetryBackoffBound, retryTimeout, - replicationListener + replicationListener, + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java index 74070e8673115..d04ed599d1e47 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.CancellableThreads; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.seqno.ReplicationTracker; @@ -334,7 +335,14 @@ private void executeRetryableAction( final ActionListener removeListener = ActionListener.runBefore(actionListener, () -> onGoingRetryableActions.remove(key)); final TimeValue initialDelay = TimeValue.timeValueMillis(200); final TimeValue timeout = recoverySettings.internalActionRetryTimeout(); - final RetryableAction retryableAction = new RetryableAction<>(logger, threadPool, initialDelay, timeout, removeListener) { + final RetryableAction retryableAction = new RetryableAction<>( + logger, + threadPool, + initialDelay, + timeout, + removeListener, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ) { @Override public void tryAction(ActionListener listener) { diff --git a/server/src/test/java/org/elasticsearch/action/support/RetryableActionTests.java b/server/src/test/java/org/elasticsearch/action/support/RetryableActionTests.java index db1a5977ffa5a..fd1439f23240e 100644 --- a/server/src/test/java/org/elasticsearch/action/support/RetryableActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/RetryableActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; @@ -38,7 +39,8 @@ public void testRetryableActionNoRetries() { taskQueue.getThreadPool(), TimeValue.timeValueMillis(10), TimeValue.timeValueSeconds(30), - future + future, + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override @@ -69,7 +71,8 @@ public void testRetryableActionWillRetry() { taskQueue.getThreadPool(), TimeValue.timeValueMillis(10), TimeValue.timeValueSeconds(30), - future + future, + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override @@ -116,7 +119,8 @@ public void testRetryableActionTimeout() { taskQueue.getThreadPool(), TimeValue.timeValueMillis(randomFrom(1, 10, randomIntBetween(100, 2000))), TimeValue.timeValueSeconds(1), - future + future, + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override @@ -163,7 +167,8 @@ public void testTimeoutOfZeroMeansNoRetry() { taskQueue.getThreadPool(), TimeValue.timeValueMillis(10), TimeValue.timeValueSeconds(0), - future + future, + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override @@ -192,7 +197,8 @@ public void testFailedBecauseNotRetryable() { taskQueue.getThreadPool(), TimeValue.timeValueMillis(10), TimeValue.timeValueSeconds(30), - future + future, + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override @@ -221,7 +227,8 @@ public void testRetryableActionCancelled() { taskQueue.getThreadPool(), TimeValue.timeValueMillis(10), TimeValue.timeValueSeconds(30), - future + future, + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override @@ -259,7 +266,8 @@ public void testMaxDelayBound() { TimeValue.timeValueMillis(10), TimeValue.timeValueMillis(50), TimeValue.timeValueSeconds(1), - future + future, + EsExecutors.DIRECT_EXECUTOR_SERVICE ) { @Override diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/PendingReplicationActionsTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/PendingReplicationActionsTests.java index 7bfc09fba0b70..73c1c1248d624 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/PendingReplicationActionsTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/PendingReplicationActionsTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.RetryableAction; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.shard.IndexShardClosedException; import org.elasticsearch.index.shard.ShardId; @@ -91,7 +92,14 @@ private TestAction(ActionListener listener) { } private TestAction(ActionListener listener, boolean succeed) { - super(logger, threadPool, TimeValue.timeValueMillis(1), TimeValue.timeValueMinutes(1), listener); + super( + logger, + threadPool, + TimeValue.timeValueMillis(1), + TimeValue.timeValueMinutes(1), + listener, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); this.succeed = succeed; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java index 0fef2a34a3de3..b19c0fb670a59 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java @@ -486,7 +486,7 @@ private RevertToCurrentSnapshotAction(JobTask jobTask, ActionListener l // to be available so that and data deletion can succeed. TimeValue.timeValueMinutes(15), listener, - MachineLearning.UTILITY_THREAD_POOL_NAME + client.threadPool().executor(MachineLearning.UTILITY_THREAD_POOL_NAME) ); this.jobTask = Objects.requireNonNull(jobTask); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java index de418364732e0..ddcfcb7de08d4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java @@ -469,7 +469,7 @@ private abstract class MlRetryableAction extends RetryableAct TimeValue.timeValueMillis(MIN_RETRY_SLEEP_MILLIS), TimeValue.MAX_VALUE, listener, - UTILITY_THREAD_POOL_NAME + threadPool.executor(UTILITY_THREAD_POOL_NAME) ); this.jobId = jobId; this.shouldRetry = shouldRetry; From 9f5ff0cfdacf8a95c47e9b3b6be46e272860ce30 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 7 Sep 2023 20:35:32 +0200 Subject: [PATCH 08/69] Forward port 8.9.2 release notes (#99323) --- docs/reference/release-notes.asciidoc | 2 ++ docs/reference/release-notes/8.9.2.asciidoc | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 docs/reference/release-notes/8.9.2.asciidoc diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index f8a602aeeea63..df8c167a1ae88 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. * <> * <> +* <> * <> * <> * <> @@ -50,6 +51,7 @@ This section summarizes the changes in each release. include::release-notes/8.11.0.asciidoc[] include::release-notes/8.10.0.asciidoc[] +include::release-notes/8.9.2.asciidoc[] include::release-notes/8.9.1.asciidoc[] include::release-notes/8.9.0.asciidoc[] include::release-notes/8.8.2.asciidoc[] diff --git a/docs/reference/release-notes/8.9.2.asciidoc b/docs/reference/release-notes/8.9.2.asciidoc new file mode 100644 index 0000000000000..d4244eab27645 --- /dev/null +++ b/docs/reference/release-notes/8.9.2.asciidoc @@ -0,0 +1,19 @@ +[[release-notes-8.9.2]] +== {es} version 8.9.2 + +Also see <>. + +[[bug-8.9.2]] +[float] +=== Bug fixes + +Data streams:: +* Avoid lifecycle NPE in the data stream lifecycle usage API {es-pull}98260[#98260] + +Geo:: +* Fix mvt error when returning partial results {es-pull}98765[#98765] (issue: {es-issue}98730[#98730]) + +Ingest Node:: +* Revert "Add mappings for enrich fields" {es-pull}98683[#98683] + + From 4b41b17772382edd074d6895747029d96ea3de4d Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Thu, 7 Sep 2023 20:47:07 +0200 Subject: [PATCH 09/69] Update documentation for Set Processor (#99191) --- docs/reference/ingest/processors/set.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ingest/processors/set.asciidoc b/docs/reference/ingest/processors/set.asciidoc index 9543e94fa14cf..7db8c62071682 100644 --- a/docs/reference/ingest/processors/set.asciidoc +++ b/docs/reference/ingest/processors/set.asciidoc @@ -16,7 +16,7 @@ its value will be replaced with the provided one. | `value` | yes* | - | The value to be set for the field. Supports <>. May specify only one of `value` or `copy_from`. | `copy_from` | no | - | The origin field which will be copied to `field`, cannot set `value` simultaneously. Supported data types are `boolean`, `number`, `array`, `object`, `string`, `date`, etc. | `override` | no | `true` | If `true` processor will update fields with pre-existing non-null-valued field. When set to `false`, such fields will not be touched. -| `ignore_empty_value` | no | `false` | If `true` and `value` is a <> that evaluates to `null` or the empty string, the processor quietly exits without modifying the document +| `ignore_empty_value` | no | `false` | If `true` and used in combination with `value` which is a <> that evaluates to `null` or an empty string, the processor quietly exits without modifying the document. Similarly, if used in combination with `copy_from` it will quietly exit if the field does not exist or its value evaluates to `null` or an empty string. | `media_type` | no | `application/json` | The media type for encoding `value`. Applies only when `value` is a <>. Must be one of `application/json`, `text/plain`, or `application/x-www-form-urlencoded`. include::common-options.asciidoc[] |====== From 676e753e6effce9158d8c02190b70c9c1dc41bfe Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 7 Sep 2023 13:17:00 -0600 Subject: [PATCH 10/69] Use deep equality when comparing IngestDocument equality (#97210) --- .../org/elasticsearch/ingest/IngestDocument.java | 2 +- .../main/java/org/elasticsearch/script/CtxMap.java | 5 +++-- .../elasticsearch/ingest/IngestDocumentTests.java | 12 ++++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 676a347c5890c..25827018e44a7 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -908,7 +908,7 @@ public boolean equals(Object obj) { } IngestDocument other = (IngestDocument) obj; - return Objects.equals(ctxMap, other.ctxMap) && Objects.equals(ingestMetadata, other.ingestMetadata); + return Objects.equals(ctxMap, other.ctxMap) && Maps.deepEquals(ingestMetadata, other.ingestMetadata); } @Override diff --git a/server/src/main/java/org/elasticsearch/script/CtxMap.java b/server/src/main/java/org/elasticsearch/script/CtxMap.java index 71c8ebea3fd36..a379ba3db4faf 100644 --- a/server/src/main/java/org/elasticsearch/script/CtxMap.java +++ b/server/src/main/java/org/elasticsearch/script/CtxMap.java @@ -8,6 +8,7 @@ package org.elasticsearch.script; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; import java.util.AbstractCollection; @@ -336,9 +337,9 @@ public Object setValue(Object value) { public boolean equals(Object o) { if (this == o) return true; if ((o instanceof CtxMap) == false) return false; - if (super.equals(o) == false) return false; CtxMap ctxMap = (CtxMap) o; - return source.equals(ctxMap.source) && metadata.equals(ctxMap.metadata); + if (Maps.deepEquals(this, ctxMap) == false) return false; + return Maps.deepEquals(source, ctxMap.source) && metadata.equals(ctxMap.metadata); } @Override diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java index 38c984c3de933..924ca1fc7a1e9 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java @@ -1158,4 +1158,16 @@ public void testIndexHistory() { assertFalse(ingestDocument.updateIndexHistory(index1)); assertThat(ingestDocument.getIndexHistory(), Matchers.contains(index1, index2)); } + + public void testEqualsAndHashCodeWithArray() { + // Test that equality still works when the ingest document uses primitive arrays, + // since normal .equals() methods would not work for Maps containing these arrays. + byte[] numbers = new byte[] { 0, 1, 2 }; + ingestDocument.setFieldValue("some.nested.array", numbers); + IngestDocument copy = new IngestDocument(ingestDocument); + byte[] copiedNumbers = copy.getFieldValue("some.nested.array", byte[].class); + assertArrayEquals(numbers, copiedNumbers); + assertNotEquals(numbers, copiedNumbers); + assertThat(copy, equalTo(ingestDocument)); + } } From 9b0dda53acd68669fe5e6b1e60743a7fa1988437 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 7 Sep 2023 14:41:15 -0700 Subject: [PATCH 11/69] Fix PIT when resolving with deleted indices (#99281) * Fix PIT when resolving with deleted indices * Update docs/changelog/99281.yaml --- docs/changelog/99281.yaml | 5 +++++ .../action/search/TransportSearchAction.java | 8 ++++---- .../action/search/TransportSearchActionTests.java | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 docs/changelog/99281.yaml diff --git a/docs/changelog/99281.yaml b/docs/changelog/99281.yaml new file mode 100644 index 0000000000000..12d018c63a863 --- /dev/null +++ b/docs/changelog/99281.yaml @@ -0,0 +1,5 @@ +pr: 99281 +summary: Fix PIT when resolving with deleted indices +area: Search +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 645418ce33e37..4cf07263fc70d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -1543,12 +1543,12 @@ static List getLocalLocalShardsIteratorFromPointInTime( if (Strings.isEmpty(perNode.getClusterAlias())) { final ShardId shardId = entry.getKey(); final List targetNodes = new ArrayList<>(2); - // Prefer executing shard requests on nodes that are part of PIT first. - if (clusterState.nodes().nodeExists(perNode.getNode())) { - targetNodes.add(perNode.getNode()); - } try { final ShardIterator shards = OperationRouting.getShards(clusterState, shardId); + // Prefer executing shard requests on nodes that are part of PIT first. + if (clusterState.nodes().nodeExists(perNode.getNode())) { + targetNodes.add(perNode.getNode()); + } if (perNode.getSearchContextId().getSearcherId() != null) { for (ShardRouting shard : shards) { if (shard.currentNodeId().equals(perNode.getNode()) == false) { diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 2eb206ed61cc0..dae91fe883a30 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -1485,7 +1485,7 @@ public void testLocalShardIteratorFromPointInTime() { true ).stream().filter(si -> si.shardId().equals(anotherShardId)).findFirst(); assertTrue(anotherShardIterator.isPresent()); - assertThat(anotherShardIterator.get().getTargetNodeIds(), hasSize(1)); + assertThat(anotherShardIterator.get().getTargetNodeIds(), hasSize(0)); } public void testCCSCompatibilityCheck() throws Exception { From b321351dd7512fd6f0f6a5c5bafbec1f3572070e Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 7 Sep 2023 23:42:27 +0200 Subject: [PATCH 12/69] Remove ActionPlugin.getClientActions (#97709) This isn't used any longer with the transport client gone. --- .../elasticsearch/plugins/ActionPlugin.java | 8 - .../xpack/core/XPackClientPlugin.java | 294 ------------------ 2 files changed, 302 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java b/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java index 207920cd60ac9..095eec2811edc 100644 --- a/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java @@ -52,14 +52,6 @@ public interface ActionPlugin { return Collections.emptyList(); } - /** - * Client actions added by this plugin. This defaults to all of the {@linkplain ActionType} in - * {@linkplain ActionPlugin#getActions()}. - */ - default List> getClientActions() { - return getActions().stream().>map(a -> a.action).toList(); - } - /** * ActionType filters added by this plugin. */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 7e2c9dd494086..5610b18ac627c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -6,21 +6,12 @@ */ package org.elasticsearch.xpack.core; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.ActionType; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.license.DeleteLicenseAction; -import org.elasticsearch.license.GetBasicStatusAction; -import org.elasticsearch.license.GetLicenseAction; -import org.elasticsearch.license.GetTrialStatusAction; import org.elasticsearch.license.LicensesMetadata; -import org.elasticsearch.license.PostStartBasicAction; -import org.elasticsearch.license.PostStartTrialAction; -import org.elasticsearch.license.PutLicenseAction; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.persistent.PersistentTaskState; import org.elasticsearch.plugins.ActionPlugin; @@ -29,27 +20,20 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersAction; -import org.elasticsearch.xpack.core.action.XPackInfoAction; -import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.aggregatemetric.AggregateMetricFeatureSetUsage; import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; import org.elasticsearch.xpack.core.application.EnterpriseSearchFeatureSetUsage; import org.elasticsearch.xpack.core.archive.ArchiveFeatureSetUsage; -import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.datastreams.DataStreamFeatureSetUsage; import org.elasticsearch.xpack.core.datastreams.DataStreamLifecycleFeatureSetUsage; -import org.elasticsearch.xpack.core.downsample.DownsampleIndexerAction; import org.elasticsearch.xpack.core.downsample.DownsampleShardStatus; import org.elasticsearch.xpack.core.enrich.EnrichFeatureSetUsage; import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyStatus; import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage; import org.elasticsearch.xpack.core.esql.EsqlFeatureSetUsage; import org.elasticsearch.xpack.core.frozen.FrozenIndicesFeatureSetUsage; -import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage; -import org.elasticsearch.xpack.core.graph.action.GraphExploreAction; import org.elasticsearch.xpack.core.ilm.AllocateAction; import org.elasticsearch.xpack.core.ilm.DeleteAction; import org.elasticsearch.xpack.core.ilm.DownsampleAction; @@ -69,79 +53,13 @@ import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; import org.elasticsearch.xpack.core.ilm.UnfollowAction; import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; -import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction; -import org.elasticsearch.xpack.core.ilm.action.ExplainLifecycleAction; -import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction; -import org.elasticsearch.xpack.core.ilm.action.MoveToStepAction; -import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; -import org.elasticsearch.xpack.core.ilm.action.RemoveIndexLifecyclePolicyAction; -import org.elasticsearch.xpack.core.ilm.action.RetryAction; import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage; import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlTasks; -import org.elasticsearch.xpack.core.ml.action.CloseJobAction; -import org.elasticsearch.xpack.core.ml.action.DeleteCalendarAction; -import org.elasticsearch.xpack.core.ml.action.DeleteCalendarEventAction; -import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction; -import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.DeleteExpiredDataAction; -import org.elasticsearch.xpack.core.ml.action.DeleteFilterAction; -import org.elasticsearch.xpack.core.ml.action.DeleteForecastAction; -import org.elasticsearch.xpack.core.ml.action.DeleteJobAction; -import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction; -import org.elasticsearch.xpack.core.ml.action.DeleteTrainedModelAction; -import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction; -import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction; -import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; -import org.elasticsearch.xpack.core.ml.action.FlushJobAction; -import org.elasticsearch.xpack.core.ml.action.ForecastJobAction; -import org.elasticsearch.xpack.core.ml.action.GetBucketsAction; -import org.elasticsearch.xpack.core.ml.action.GetCalendarEventsAction; -import org.elasticsearch.xpack.core.ml.action.GetCalendarsAction; -import org.elasticsearch.xpack.core.ml.action.GetCategoriesAction; -import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction; -import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction; -import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; -import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; -import org.elasticsearch.xpack.core.ml.action.GetFiltersAction; -import org.elasticsearch.xpack.core.ml.action.GetInfluencersAction; -import org.elasticsearch.xpack.core.ml.action.GetJobsAction; -import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; -import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction; -import org.elasticsearch.xpack.core.ml.action.GetOverallBucketsAction; -import org.elasticsearch.xpack.core.ml.action.GetRecordsAction; -import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; -import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsAction; -import org.elasticsearch.xpack.core.ml.action.InferModelAction; -import org.elasticsearch.xpack.core.ml.action.IsolateDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.KillProcessAction; -import org.elasticsearch.xpack.core.ml.action.MlInfoAction; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; -import org.elasticsearch.xpack.core.ml.action.PersistJobAction; -import org.elasticsearch.xpack.core.ml.action.PostCalendarEventsAction; -import org.elasticsearch.xpack.core.ml.action.PostDataAction; -import org.elasticsearch.xpack.core.ml.action.PreviewDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.PutCalendarAction; -import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction; -import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.PutFilterAction; -import org.elasticsearch.xpack.core.ml.action.PutJobAction; -import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction; -import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction; -import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction; import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.UpdateCalendarJobAction; -import org.elasticsearch.xpack.core.ml.action.UpdateDataFrameAnalyticsAction; -import org.elasticsearch.xpack.core.ml.action.UpdateDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.UpdateFilterAction; -import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; -import org.elasticsearch.xpack.core.ml.action.UpdateModelSnapshotAction; -import org.elasticsearch.xpack.core.ml.action.UpdateProcessAction; -import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction; -import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; @@ -150,40 +68,10 @@ import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage; import org.elasticsearch.xpack.core.rollup.RollupFeatureSetUsage; import org.elasticsearch.xpack.core.rollup.RollupField; -import org.elasticsearch.xpack.core.rollup.action.DeleteRollupJobAction; -import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction; -import org.elasticsearch.xpack.core.rollup.action.GetRollupJobsAction; -import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction; -import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction; -import org.elasticsearch.xpack.core.rollup.action.StartRollupJobAction; -import org.elasticsearch.xpack.core.rollup.action.StopRollupJobAction; import org.elasticsearch.xpack.core.rollup.job.RollupJob; import org.elasticsearch.xpack.core.rollup.job.RollupJobStatus; -import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; -import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotFeatureSetUsage; import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage; -import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyAction; -import org.elasticsearch.xpack.core.security.action.apikey.GetApiKeyAction; -import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyAction; -import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction; -import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheAction; -import org.elasticsearch.xpack.core.security.action.role.DeleteRoleAction; -import org.elasticsearch.xpack.core.security.action.role.GetRolesAction; -import org.elasticsearch.xpack.core.security.action.role.PutRoleAction; -import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction; -import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction; -import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction; -import org.elasticsearch.xpack.core.security.action.token.CreateTokenAction; -import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenAction; -import org.elasticsearch.xpack.core.security.action.token.RefreshTokenAction; -import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction; -import org.elasticsearch.xpack.core.security.action.user.ChangePasswordAction; -import org.elasticsearch.xpack.core.security.action.user.DeleteUserAction; -import org.elasticsearch.xpack.core.security.action.user.GetUsersAction; -import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction; -import org.elasticsearch.xpack.core.security.action.user.PutUserAction; -import org.elasticsearch.xpack.core.security.action.user.SetEnabledAction; import org.elasticsearch.xpack.core.security.authc.TokenMetadata; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.AllExpression; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.AnyExpression; @@ -194,26 +82,11 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges; import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; -import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction; -import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction; -import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleAction; -import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleStatsAction; -import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction; import org.elasticsearch.xpack.core.spatial.SpatialFeatureSetUsage; import org.elasticsearch.xpack.core.sql.SqlFeatureSetUsage; -import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction; -import org.elasticsearch.xpack.core.termsenum.action.TermsEnumAction; -import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction; import org.elasticsearch.xpack.core.transform.TransformFeatureSetUsage; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.TransformMetadata; -import org.elasticsearch.xpack.core.transform.action.DeleteTransformAction; -import org.elasticsearch.xpack.core.transform.action.GetTransformAction; -import org.elasticsearch.xpack.core.transform.action.GetTransformStatsAction; -import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction; -import org.elasticsearch.xpack.core.transform.action.PutTransformAction; -import org.elasticsearch.xpack.core.transform.action.StartTransformAction; -import org.elasticsearch.xpack.core.transform.action.StopTransformAction; import org.elasticsearch.xpack.core.transform.transforms.NullRetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; @@ -224,14 +97,6 @@ import org.elasticsearch.xpack.core.votingonly.VotingOnlyNodeFeatureSetUsage; import org.elasticsearch.xpack.core.watcher.WatcherFeatureSetUsage; import org.elasticsearch.xpack.core.watcher.WatcherMetadata; -import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchAction; -import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchAction; -import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchAction; -import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchAction; -import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchAction; -import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction; -import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction; -import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction; import java.util.ArrayList; import java.util.Arrays; @@ -257,165 +122,6 @@ public List> getSettings() { return settings; } - @Override - public List> getClientActions() { - return Arrays.asList( - // graph - GraphExploreAction.INSTANCE, - // ML - GetJobsAction.INSTANCE, - GetJobsStatsAction.INSTANCE, - MlInfoAction.INSTANCE, - PutJobAction.INSTANCE, - UpdateJobAction.INSTANCE, - DeleteJobAction.INSTANCE, - OpenJobAction.INSTANCE, - GetFiltersAction.INSTANCE, - PutFilterAction.INSTANCE, - UpdateFilterAction.INSTANCE, - DeleteFilterAction.INSTANCE, - KillProcessAction.INSTANCE, - GetBucketsAction.INSTANCE, - GetInfluencersAction.INSTANCE, - GetOverallBucketsAction.INSTANCE, - GetRecordsAction.INSTANCE, - PostDataAction.INSTANCE, - CloseJobAction.INSTANCE, - FinalizeJobExecutionAction.INSTANCE, - FlushJobAction.INSTANCE, - ValidateDetectorAction.INSTANCE, - ValidateJobConfigAction.INSTANCE, - GetCategoriesAction.INSTANCE, - GetModelSnapshotsAction.INSTANCE, - RevertModelSnapshotAction.INSTANCE, - UpdateModelSnapshotAction.INSTANCE, - GetDatafeedsAction.INSTANCE, - GetDatafeedsStatsAction.INSTANCE, - PutDatafeedAction.INSTANCE, - UpdateDatafeedAction.INSTANCE, - DeleteDatafeedAction.INSTANCE, - PreviewDatafeedAction.INSTANCE, - StartDatafeedAction.INSTANCE, - StopDatafeedAction.INSTANCE, - IsolateDatafeedAction.INSTANCE, - DeleteModelSnapshotAction.INSTANCE, - UpdateProcessAction.INSTANCE, - DeleteExpiredDataAction.INSTANCE, - ForecastJobAction.INSTANCE, - DeleteForecastAction.INSTANCE, - GetCalendarsAction.INSTANCE, - PutCalendarAction.INSTANCE, - DeleteCalendarAction.INSTANCE, - DeleteCalendarEventAction.INSTANCE, - UpdateCalendarJobAction.INSTANCE, - GetCalendarEventsAction.INSTANCE, - PostCalendarEventsAction.INSTANCE, - PersistJobAction.INSTANCE, - SetUpgradeModeAction.INSTANCE, - PutDataFrameAnalyticsAction.INSTANCE, - GetDataFrameAnalyticsAction.INSTANCE, - GetDataFrameAnalyticsStatsAction.INSTANCE, - UpdateDataFrameAnalyticsAction.INSTANCE, - DeleteDataFrameAnalyticsAction.INSTANCE, - StartDataFrameAnalyticsAction.INSTANCE, - EvaluateDataFrameAction.INSTANCE, - ExplainDataFrameAnalyticsAction.INSTANCE, - InferModelAction.INSTANCE, - InferModelAction.EXTERNAL_INSTANCE, - GetTrainedModelsAction.INSTANCE, - DeleteTrainedModelAction.INSTANCE, - GetTrainedModelsStatsAction.INSTANCE, - PutTrainedModelAction.INSTANCE, - // security - ClearRealmCacheAction.INSTANCE, - ClearRolesCacheAction.INSTANCE, - GetUsersAction.INSTANCE, - PutUserAction.INSTANCE, - DeleteUserAction.INSTANCE, - GetRolesAction.INSTANCE, - PutRoleAction.INSTANCE, - DeleteRoleAction.INSTANCE, - ChangePasswordAction.INSTANCE, - AuthenticateAction.INSTANCE, - SetEnabledAction.INSTANCE, - HasPrivilegesAction.INSTANCE, - GetRoleMappingsAction.INSTANCE, - PutRoleMappingAction.INSTANCE, - DeleteRoleMappingAction.INSTANCE, - CreateTokenAction.INSTANCE, - InvalidateTokenAction.INSTANCE, - GetCertificateInfoAction.INSTANCE, - RefreshTokenAction.INSTANCE, - CreateApiKeyAction.INSTANCE, - InvalidateApiKeyAction.INSTANCE, - GetApiKeyAction.INSTANCE, - // watcher - PutWatchAction.INSTANCE, - DeleteWatchAction.INSTANCE, - GetWatchAction.INSTANCE, - WatcherStatsAction.INSTANCE, - AckWatchAction.INSTANCE, - ActivateWatchAction.INSTANCE, - WatcherServiceAction.INSTANCE, - ExecuteWatchAction.INSTANCE, - // license - PutLicenseAction.INSTANCE, - GetLicenseAction.INSTANCE, - DeleteLicenseAction.INSTANCE, - PostStartTrialAction.INSTANCE, - GetTrialStatusAction.INSTANCE, - PostStartBasicAction.INSTANCE, - GetBasicStatusAction.INSTANCE, - // x-pack - XPackInfoAction.INSTANCE, - XPackUsageAction.INSTANCE, - // rollup - RollupSearchAction.INSTANCE, - PutRollupJobAction.INSTANCE, - StartRollupJobAction.INSTANCE, - StopRollupJobAction.INSTANCE, - DeleteRollupJobAction.INSTANCE, - GetRollupJobsAction.INSTANCE, - GetRollupCapsAction.INSTANCE, - // ILM - DeleteLifecycleAction.INSTANCE, - GetLifecycleAction.INSTANCE, - PutLifecycleAction.INSTANCE, - ExplainLifecycleAction.INSTANCE, - RemoveIndexLifecyclePolicyAction.INSTANCE, - MoveToStepAction.INSTANCE, - RetryAction.INSTANCE, - PutSnapshotLifecycleAction.INSTANCE, - GetSnapshotLifecycleAction.INSTANCE, - DeleteSnapshotLifecycleAction.INSTANCE, - ExecuteSnapshotLifecycleAction.INSTANCE, - GetSnapshotLifecycleStatsAction.INSTANCE, - MigrateToDataTiersAction.INSTANCE, - - // Freeze - FreezeIndexAction.INSTANCE, - // Data Frame - PutTransformAction.INSTANCE, - StartTransformAction.INSTANCE, - StopTransformAction.INSTANCE, - DeleteTransformAction.INSTANCE, - GetTransformAction.INSTANCE, - GetTransformStatsAction.INSTANCE, - PreviewTransformAction.INSTANCE, - // Async Search - SubmitAsyncSearchAction.INSTANCE, - GetAsyncSearchAction.INSTANCE, - DeleteAsyncResultAction.INSTANCE, - // Text Structure - FindStructureAction.INSTANCE, - // Terms enum API - TermsEnumAction.INSTANCE, - // TSDB Downsampling - DownsampleIndexerAction.INSTANCE, - org.elasticsearch.action.downsample.DownsampleAction.INSTANCE - ); - } - @Override public List getNamedWriteables() { return Stream.of( From e8f8a0ac2eee36588e97c9a4eed9f64e25f33f78 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Fri, 8 Sep 2023 12:32:47 +1000 Subject: [PATCH 13/69] [IdP] Fix Idp Builder using method calls (#99221) The `SamlIdentityProviderBuilder` has mechanisms for building an object using direct method calls, or from a `Settings` object. Due to a lack of tests there was a bug that prevented the direct method call approach from working correctly. --- .../saml/idp/SamlIdentityProviderBuilder.java | 7 ++++ .../idp/SamlIdentityProviderBuilderTests.java | 40 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java index 885e77d3eeec0..8051b08c78a86 100644 --- a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java +++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilder.java @@ -168,6 +168,10 @@ public SamlIdentityProvider build() throws ValidationException { ex.addValidationError("Service provider defaults must be specified"); } + if (allowedNameIdFormats == null || allowedNameIdFormats.isEmpty()) { + ex.addValidationError("At least 1 allowed NameID format must be specified"); + } + if (ex.validationErrors().isEmpty() == false) { throw ex; } @@ -260,6 +264,9 @@ public SamlIdentityProviderBuilder singleLogoutEndpoint(String binding, URL endp } public SamlIdentityProviderBuilder allowedNameIdFormat(String nameIdFormat) { + if (this.allowedNameIdFormats == null) { + this.allowedNameIdFormats = new HashSet<>(); + } this.allowedNameIdFormats.add(nameIdFormat); return this; } diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java index 1ca0a955dd08a..e5f995204ac0c 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/idp/SamlIdentityProviderBuilderTests.java @@ -21,8 +21,10 @@ import org.elasticsearch.xpack.idp.saml.test.IdpSamlTestCase; import org.hamcrest.Matchers; import org.mockito.Mockito; +import org.opensaml.saml.saml2.core.NameID; import org.opensaml.security.x509.X509Credential; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.security.PrivateKey; @@ -43,13 +45,16 @@ import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SLO_REDIRECT_ENDPOINT; import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SSO_POST_ENDPOINT; import static org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProviderBuilder.IDP_SSO_REDIRECT_ENDPOINT; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_POST_BINDING_URI; import static org.opensaml.saml.common.xml.SAMLConstants.SAML2_REDIRECT_BINDING_URI; +import static org.opensaml.saml.saml2.core.NameIDType.EMAIL; import static org.opensaml.saml.saml2.core.NameIDType.PERSISTENT; import static org.opensaml.saml.saml2.core.NameIDType.TRANSIENT; @@ -592,4 +597,39 @@ public void testCreateMetadataSigningCredentialFromKeyStoreWithMultipleEntriesBu ); } + public void testCreateViaMethodCalls() throws Exception { + final String entityId = randomAlphaOfLength(4) + ":" + randomAlphaOfLength(6) + "/" + randomAlphaOfLengthBetween(4, 12); + final URL redirectUrl = new URL( + randomFrom("http", "https") + + "://" + + String.join(".", randomArray(2, 5, String[]::new, () -> randomAlphaOfLengthBetween(3, 6))) + + "/" + + String.join("/", randomArray(1, 3, String[]::new, () -> randomAlphaOfLengthBetween(2, 4))) + ); + + final X509Credential credential = readCredentials("RSA", randomFrom(1024, 2048)); + final String nameIdFormat = randomFrom(NameID.TRANSIENT, PERSISTENT, EMAIL); + + final SamlServiceProviderResolver serviceResolver = Mockito.mock(SamlServiceProviderResolver.class); + final WildcardServiceProviderResolver wildcardResolver = Mockito.mock(WildcardServiceProviderResolver.class); + final ServiceProviderDefaults spDefaults = new ServiceProviderDefaults( + randomAlphaOfLength(2), + nameIdFormat, + Duration.ofMinutes(randomIntBetween(1, 10)) + ); + final SamlIdentityProvider idp = SamlIdentityProvider.builder(serviceResolver, wildcardResolver) + .entityId(entityId) + .singleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI, redirectUrl) + .signingCredential(credential) + .serviceProviderDefaults(spDefaults) + .allowedNameIdFormat(nameIdFormat) + .build(); + + assertThat(idp.getEntityId(), is(entityId)); + assertThat(idp.getSingleSignOnEndpoint(SAML2_REDIRECT_BINDING_URI), is(redirectUrl)); + assertThat(idp.getSigningCredential(), is(credential)); + assertThat(idp.getServiceProviderDefaults(), is(spDefaults)); + assertThat(idp.getAllowedNameIdFormats(), contains(nameIdFormat)); + } + } From d3f5584b214e8a294f0b4bba74a7864c8699f3c6 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Fri, 8 Sep 2023 07:08:09 +0100 Subject: [PATCH 14/69] Fix NameId Mapper in PlanStreamInput (#99295) This commit fixes the NameId mapper in PlanStreamInput. The current mapper function just materializes a NameId instance with the stream's primitive long value. This can lead to a potential issue during re-planning if the local data node planner just happens to use an id, retrieved automatically from the global counter, during the course of its planning. This can lead to confusion, where the same id is used for different logical attributes. The updated Mapper keeps a map of seen stream Ids to NameId instances. The no-args NameId constructor is used for absent entries, as it will automatically select and increment an id from the global counter, thus avoiding potential conflicts between the id in the stream and id's during local re-planning on the data node. --- .../xpack/esql/io/stream/PlanStreamInput.java | 32 ++++--- .../esql/io/stream/PlanNamedTypesTests.java | 28 +++--- .../esql/io/stream/PlanStreamInputTests.java | 89 +++++++++++++++++++ .../xpack/ql/expression/NameId.java | 4 - 4 files changed, 123 insertions(+), 30 deletions(-) create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java index 49173779406cc..8e38700a58225 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java @@ -28,8 +28,11 @@ import java.io.IOException; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.function.LongFunction; +import java.util.function.Supplier; /** * A customized stream input used to deserialize ESQL physical plan fragments. Complements stream @@ -37,7 +40,22 @@ */ public final class PlanStreamInput extends NamedWriteableAwareStreamInput { - private static final LongFunction DEFAULT_NAME_ID_FUNC = NameId::new; + /** + * A Mapper of stream named id, represented as a primitive long value, to NameId instance. + * The no-args NameId constructor is used for absent entries, as it will automatically select + * and increment an id from the global counter, thus avoiding potential conflicts between the + * id in the stream and id's during local re-planning on the data node. + */ + static final class NameIdMapper implements LongFunction { + final Map seen = new HashMap<>(); + + @Override + public NameId apply(long streamNameId) { + return seen.computeIfAbsent(streamNameId, k -> new NameId()); + } + } + + private static final Supplier> DEFAULT_NAME_ID_FUNC = NameIdMapper::new; private final PlanNameRegistry registry; @@ -51,21 +69,11 @@ public PlanStreamInput( PlanNameRegistry registry, NamedWriteableRegistry namedWriteableRegistry, EsqlConfiguration configuration - ) { - this(streamInput, registry, namedWriteableRegistry, configuration, DEFAULT_NAME_ID_FUNC); - } - - public PlanStreamInput( - StreamInput streamInput, - PlanNameRegistry registry, - NamedWriteableRegistry namedWriteableRegistry, - EsqlConfiguration configuration, - LongFunction nameIdFunction ) { super(streamInput, namedWriteableRegistry); this.registry = registry; - this.nameIdFunction = nameIdFunction; this.configuration = configuration; + this.nameIdFunction = DEFAULT_NAME_ID_FUNC.get(); } NameId nameIdFromLongValue(long value) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java index 4a0dbf88feb47..a5cb0044d91d0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java @@ -169,20 +169,19 @@ public void testUnsupportedAttributeSimple() throws IOException { "foo", new UnsupportedEsField("foo", "keyword"), "field not supported", - new NameId(53) + new NameId() ); BytesStreamOutput bso = new BytesStreamOutput(); PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry); PlanNamedTypes.writeUnsupportedAttr(out, orig); - var deser = PlanNamedTypes.readUnsupportedAttr(planStreamInput(bso)); + var in = planStreamInput(bso); + var deser = PlanNamedTypes.readUnsupportedAttr(in); EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); - assertThat(deser.id(), equalTo(orig.id())); + assertThat(deser.id(), equalTo(in.nameIdFromLongValue(Long.parseLong(orig.id().toString())))); } public void testUnsupportedAttribute() { - Stream.generate(PlanNamedTypesTests::randomUnsupportedAttribute) - .limit(100) - .forEach(PlanNamedTypesTests::assertNamedExpressionAndId); + Stream.generate(PlanNamedTypesTests::randomUnsupportedAttribute).limit(100).forEach(PlanNamedTypesTests::assertNamedExpression); } public void testFieldAttributeSimple() throws IOException { @@ -194,19 +193,20 @@ public void testFieldAttributeSimple() throws IOException { randomEsField(), null, // qualifier, can be null Nullability.TRUE, - new NameId(53), + new NameId(), true // synthetic ); BytesStreamOutput bso = new BytesStreamOutput(); PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry); PlanNamedTypes.writeFieldAttribute(out, orig); - var deser = PlanNamedTypes.readFieldAttribute(planStreamInput(bso)); + var in = planStreamInput(bso); + var deser = PlanNamedTypes.readFieldAttribute(in); EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); - assertThat(deser.id(), equalTo(orig.id())); + assertThat(deser.id(), equalTo(in.nameIdFromLongValue(Long.parseLong(orig.id().toString())))); } public void testFieldAttribute() { - Stream.generate(PlanNamedTypesTests::randomFieldAttribute).limit(100).forEach(PlanNamedTypesTests::assertNamedExpressionAndId); + Stream.generate(PlanNamedTypesTests::randomFieldAttribute).limit(100).forEach(PlanNamedTypesTests::assertNamedExpression); } public void testKeywordEsFieldSimple() throws IOException { @@ -353,9 +353,10 @@ public void testAliasSimple() throws IOException { BytesStreamOutput bso = new BytesStreamOutput(); PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry); PlanNamedTypes.writeAlias(out, orig); - var deser = PlanNamedTypes.readAlias(planStreamInput(bso)); + var in = planStreamInput(bso); + var deser = PlanNamedTypes.readAlias(in); EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); - assertThat(orig.id(), equalTo(deser.id())); + assertThat(deser.id(), equalTo(in.nameIdFromLongValue(Long.parseLong(orig.id().toString())))); } public void testLiteralSimple() throws IOException { @@ -404,10 +405,9 @@ public void testDissectParserSimple() throws IOException { EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); } - private static void assertNamedExpressionAndId(NamedExpression origObj) { + private static void assertNamedExpression(NamedExpression origObj) { var deserObj = serializeDeserialize(origObj, PlanStreamOutput::writeExpression, PlanStreamInput::readNamedExpression); EqualsHashCodeTestUtils.checkEqualsAndHashCode(origObj, unused -> deserObj); - assertThat(deserObj.id(), equalTo(origObj.id())); } private static void assertNamedType(Class type, T origObj) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java new file mode 100644 index 0000000000000..fe885931d3114 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInputTests.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.io.stream; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ql.expression.NameId; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +public class PlanStreamInputTests extends ESTestCase { + + public void testMapperSimple() { + var mapper = new PlanStreamInput.NameIdMapper(); + + NameId first = mapper.apply(1L); + NameId second = mapper.apply(1L); + assertThat(second, equalTo(first)); + + NameId third = mapper.apply(2L); + NameId fourth = mapper.apply(2L); + assertThat(third, not(equalTo(second))); + assertThat(fourth, equalTo(third)); + + assertThat(mapper.seen.size(), is(2)); + } + + public void testMapper() { + List longs = randomLongsListOfSize(100); + List nameIds = new ArrayList<>(); + for (long l : longs) { + nameIds.add(l); + if (randomBoolean()) { // randomly insert additional values from the known list + int idx = randomIntBetween(0, longs.size() - 1); + nameIds.add(longs.get(idx)); + } + } + + var mapper = new PlanStreamInput.NameIdMapper(); + List mappedIds = nameIds.stream().map(mapper::apply).toList(); + assertThat(mappedIds.size(), is(nameIds.size())); + // there must be exactly 100 distinct elements + assertThat(mapper.seen.size(), is(100)); + assertThat(mappedIds.stream().distinct().count(), is(100L)); + + // The pre-mapped name id pattern must match that of the mapped one + Map> nameIdsSeen = new LinkedHashMap<>(); // insertion order + for (int i = 0; i < nameIds.size(); i++) { + long value = nameIds.get(i); + nameIdsSeen.computeIfAbsent(value, k -> new ArrayList<>()); + nameIdsSeen.get(value).add((long) i); + } + assert nameIdsSeen.size() == 100; + + Map> mappedSeen = new LinkedHashMap<>(); // insertion order + for (int i = 0; i < mappedIds.size(); i++) { + NameId nameId = mappedIds.get(i); + mappedSeen.computeIfAbsent(nameId, k -> new ArrayList<>()); + mappedSeen.get(nameId).add((long) i); + } + assert mappedSeen.size() == 100; + + var mappedSeenItr = mappedSeen.values().iterator(); + for (List indexes : nameIdsSeen.values()) { + assertThat(indexes, equalTo(mappedSeenItr.next())); + } + } + + List randomLongsListOfSize(int size) { + Set longs = new HashSet<>(); + while (longs.size() < size) { + longs.add(randomLong()); + } + return longs.stream().toList(); + } +} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/NameId.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/NameId.java index e099621756e37..20e8214ddef34 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/NameId.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/NameId.java @@ -26,10 +26,6 @@ public NameId() { this.id = COUNTER.incrementAndGet(); } - public NameId(long id) { - this.id = id; - } - @Override public int hashCode() { return Objects.hash(id); From ca0068c2fc7fca763f615e276ee79a96d4248e28 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Fri, 8 Sep 2023 01:22:44 -0500 Subject: [PATCH 15/69] Fix deadlock in jwt realm (#99328) This commit fixes the deadlock described in https://github.com/elastic/elasticsearch/issues/99317 for the jwt realm. The fix is to ensure that Cache#put and Cache#invalidateAll are mutually exclusive as to not trigger the deadlock that can occur in the Cache implementation. (see https://github.com/elastic/elasticsearch/issues/99326) This commit also renames the variables to make them more obvious as to what they represent. Fixes #99317 --- .../security/support/CacheIteratorHelper.java | 8 +- .../xpack/security/authc/jwt/JwtRealm.java | 5 +- .../authc/jwt/JwtRealmAuthenticateTests.java | 81 +++++++++++++++++++ 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/CacheIteratorHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/CacheIteratorHelper.java index af2fcf041172f..5b372ed91b7bf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/CacheIteratorHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/CacheIteratorHelper.java @@ -31,8 +31,10 @@ public CacheIteratorHelper(Cache cache) { final ReadWriteLock lock = new ReentrantReadWriteLock(); // the lock is used in an odd manner; when iterating over the cache we cannot have modifiers other than deletes using the // iterator but when not iterating we can modify the cache without external locking. When making normal modifications to the cache - // the read lock is obtained so that we can allow concurrent modifications; however when we need to iterate over the keys or values - // of the cache the write lock must obtained to prevent any modifications. + // the read lock can be obtained so that we can allow concurrent modifications; however when we need to iterate over the keys or + // values of the cache the write lock must be obtained to prevent any modifications. + // Note - the write lock is needed for concurrent modifications across Cache#put and Cache#invalidateAll + // see https://github.com/elastic/elasticsearch/issues/99326 for additional information updateLock = new ReleasableLock(lock.readLock()); iteratorLock = new ReleasableLock(lock.writeLock()); } @@ -41,7 +43,7 @@ public ReleasableLock acquireUpdateLock() { return updateLock.acquire(); } - private ReleasableLock acquireForIterator() { + public ReleasableLock acquireForIterator() { return iteratorLock.acquire(); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java index 943e37c8d01e3..41795bbb62010 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java @@ -372,7 +372,8 @@ private User tryAuthenticateWithCache(final String tokenPrincipal, final BytesAr return null; } - private void processValidatedJwt( + // package private for testing + void processValidatedJwt( String tokenPrincipal, BytesArray jwtCacheKey, JWTClaimsSet claimsSet, @@ -449,7 +450,7 @@ private void invalidateJwtCache() { if (isCacheEnabled()) { try { logger.trace("Invalidating JWT cache for realm [{}]", name()); - try (ReleasableLock ignored = jwtCacheHelper.acquireUpdateLock()) { + try (ReleasableLock ignored = jwtCacheHelper.acquireForIterator()) { jwtCache.invalidateAll(); } logger.debug("Invalidated JWT cache for realm [{}]", name()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java index 51c6b677b56b0..4cb510e9345f6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -32,6 +33,8 @@ import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -530,4 +533,82 @@ public void testSameIssuerTwoRealmsDifferentClientSecrets() throws Exception { final int jwtAuthcCount = randomIntBetween(2, 3); doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcCount); } + + public void testConcurrentPutAndInvalidateCacheWorks() throws Exception { + jwtIssuerAndRealms = generateJwtIssuerRealmPairs( + randomIntBetween(1, 1), // realmsRange + randomIntBetween(0, 0), // authzRange + randomIntBetween(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange + randomIntBetween(1, 1), // audiencesRange + randomIntBetween(1, 1), // usersRange + randomIntBetween(1, 1), // rolesRange + randomIntBetween(1, 1), // jwtCacheSizeRange set to 1 for constant eviction that is necessary to trigger the locking when put + false // createHttpsServer + ); + + final JwtIssuerAndRealm jwtIssuerAndRealm = randomJwtIssuerRealmPair(); + final User user = randomUser(jwtIssuerAndRealm.issuer()); + final SecureString jwt = randomJwt(jwtIssuerAndRealm, user); + final SignedJWT parsedJwt = SignedJWT.parse(jwt.toString()); + final JWTClaimsSet validClaimsSet = parsedJwt.getJWTClaimsSet(); + + final int processors = Runtime.getRuntime().availableProcessors(); + final int numberOfThreads = Math.min(50, scaledRandomIntBetween((processors + 1) / 2, 4 * processors)); // up to 50 threads + final Thread[] threads = new Thread[numberOfThreads]; + final CountDownLatch threadsCountDown = new CountDownLatch(numberOfThreads); + final CountDownLatch racingCountDown = new CountDownLatch(1); + final CountDownLatch completionCountDown = new CountDownLatch(numberOfThreads); + + for (int i = 0; i < numberOfThreads; i++) { + if (randomBoolean()) { + threads[i] = new Thread(() -> { + threadsCountDown.countDown(); + try { + if (racingCountDown.await(10, TimeUnit.SECONDS)) { + jwtIssuerAndRealm.realm().expireAll(); + completionCountDown.countDown(); + } else { + throw new AssertionError("racing is not ready within the given time period"); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + } else { + threads[i] = new Thread(() -> { + final BytesArray jwtCacheKey = new BytesArray(randomAlphaOfLength(10)); + final PlainActionFuture> future = new PlainActionFuture<>(); + threadsCountDown.countDown(); + try { + if (racingCountDown.await(10, TimeUnit.SECONDS)) { + for (int j = 0; j < 10; j++) { + jwtIssuerAndRealm.realm().processValidatedJwt("token-principal", jwtCacheKey, validClaimsSet, future); + assertThat(future.actionGet().getValue().principal(), equalTo(user.principal())); + } + completionCountDown.countDown(); + } else { + throw new AssertionError("Racing is not ready within the given time period"); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + } + threads[i].start(); + } + + if (threadsCountDown.await(10, TimeUnit.SECONDS)) { + racingCountDown.countDown(); + } else { + throw new AssertionError("Threads are not ready within the given time period"); + } + + if (false == completionCountDown.await(30, TimeUnit.SECONDS)) { + throw new AssertionError("Test is not completed in time, check whether threads had deadlock"); + } + + for (Thread thread : threads) { + thread.join(); + } + } } From cf9bb6f963bfbb71f29711a019ea120009ea1e7f Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 8 Sep 2023 08:59:46 +0200 Subject: [PATCH 16/69] Close index reader in tests (#99288) Not closing the index reader in unit tests did not cause issues so far, but it becomes a problem since we use newSearcher more often which randomly sets an executor to the searcher. This is because the executor is shutdown as part of the index reader closing listener, and if the reader is not closed, the test leaks threads which causes test failures. This is more evident in the lucene_snapshot branch as lucene 9.8 will introduce unconditional offloading of tasks to the executor when provided (including sequential execution). In main, there's many cases where even if we have an executor, it is never used because execution is sequential (single slice) hence it won't leak threads in that case. This commit fixes the unit tests that use LuceneTestCase#newSearcher to create the searcher, which may randomize the executor and need the reader to be closed as that triggers the shutdown of the executor too. --- .../painless/SimilarityScriptTests.java | 116 ++++++------- .../search/MultiPhrasePrefixQueryTests.java | 54 +++--- .../deps/lucene/SimpleLuceneTests.java | 161 +++++++++--------- 3 files changed, 171 insertions(+), 160 deletions(-) diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java index bbc24b74513f6..8437d78962c0c 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SimilarityScriptTests.java @@ -55,39 +55,39 @@ public void testBasics() throws IOException { Collections.emptyMap() ); ScriptedSimilarity sim = new ScriptedSimilarity("foobar", null, "foobaz", factory::newInstance, true); - Directory dir = new ByteBuffersDirectory(); - IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(sim)); + try (Directory dir = new ByteBuffersDirectory()) { + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(sim)); - Document doc = new Document(); - doc.add(new TextField("f", "foo bar", Store.NO)); - doc.add(new StringField("match", "no", Store.NO)); - w.addDocument(doc); + Document doc = new Document(); + doc.add(new TextField("f", "foo bar", Store.NO)); + doc.add(new StringField("match", "no", Store.NO)); + w.addDocument(doc); - doc = new Document(); - doc.add(new TextField("f", "foo foo bar", Store.NO)); - doc.add(new StringField("match", "yes", Store.NO)); - w.addDocument(doc); + doc = new Document(); + doc.add(new TextField("f", "foo foo bar", Store.NO)); + doc.add(new StringField("match", "yes", Store.NO)); + w.addDocument(doc); - doc = new Document(); - doc.add(new TextField("f", "bar", Store.NO)); - doc.add(new StringField("match", "no", Store.NO)); - w.addDocument(doc); + doc = new Document(); + doc.add(new TextField("f", "bar", Store.NO)); + doc.add(new StringField("match", "no", Store.NO)); + w.addDocument(doc); - IndexReader r = DirectoryReader.open(w); - w.close(); - IndexSearcher searcher = newSearcher(r); - searcher.setSimilarity(sim); - Query query = new BoostQuery( - new BooleanQuery.Builder().add(new TermQuery(new Term("f", "foo")), Occur.SHOULD) - .add(new TermQuery(new Term("match", "yes")), Occur.FILTER) - .build(), - 3.2f - ); - TopDocs topDocs = searcher.search(query, 1); - assertEquals(1, topDocs.totalHits.value); - assertEquals((float) (3.2 * 2 / 3), topDocs.scoreDocs[0].score, 0); - w.close(); - dir.close(); + try (IndexReader r = DirectoryReader.open(w)) { + w.close(); + IndexSearcher searcher = newSearcher(r); + searcher.setSimilarity(sim); + Query query = new BoostQuery( + new BooleanQuery.Builder().add(new TermQuery(new Term("f", "foo")), Occur.SHOULD) + .add(new TermQuery(new Term("match", "yes")), Occur.FILTER) + .build(), + 3.2f + ); + TopDocs topDocs = searcher.search(query, 1); + assertEquals(1, topDocs.totalHits.value); + assertEquals((float) (3.2 * 2 / 3), topDocs.scoreDocs[0].score, 0); + } + } } public void testWeightScript() throws IOException { @@ -104,38 +104,38 @@ public void testWeightScript() throws IOException { Collections.emptyMap() ); ScriptedSimilarity sim = new ScriptedSimilarity("foobar", weightFactory::newInstance, "foobaz", factory::newInstance, true); - Directory dir = new ByteBuffersDirectory(); - IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(sim)); + try (Directory dir = new ByteBuffersDirectory()) { + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setSimilarity(sim)); - Document doc = new Document(); - doc.add(new TextField("f", "foo bar", Store.NO)); - doc.add(new StringField("match", "no", Store.NO)); - w.addDocument(doc); + Document doc = new Document(); + doc.add(new TextField("f", "foo bar", Store.NO)); + doc.add(new StringField("match", "no", Store.NO)); + w.addDocument(doc); - doc = new Document(); - doc.add(new TextField("f", "foo foo bar", Store.NO)); - doc.add(new StringField("match", "yes", Store.NO)); - w.addDocument(doc); + doc = new Document(); + doc.add(new TextField("f", "foo foo bar", Store.NO)); + doc.add(new StringField("match", "yes", Store.NO)); + w.addDocument(doc); - doc = new Document(); - doc.add(new TextField("f", "bar", Store.NO)); - doc.add(new StringField("match", "no", Store.NO)); - w.addDocument(doc); + doc = new Document(); + doc.add(new TextField("f", "bar", Store.NO)); + doc.add(new StringField("match", "no", Store.NO)); + w.addDocument(doc); - IndexReader r = DirectoryReader.open(w); - w.close(); - IndexSearcher searcher = newSearcher(r); - searcher.setSimilarity(sim); - Query query = new BoostQuery( - new BooleanQuery.Builder().add(new TermQuery(new Term("f", "foo")), Occur.SHOULD) - .add(new TermQuery(new Term("match", "yes")), Occur.FILTER) - .build(), - 3.2f - ); - TopDocs topDocs = searcher.search(query, 1); - assertEquals(1, topDocs.totalHits.value); - assertEquals((float) (3.2 * 2 / 3), topDocs.scoreDocs[0].score, 0); - w.close(); - dir.close(); + try (IndexReader r = DirectoryReader.open(w)) { + w.close(); + IndexSearcher searcher = newSearcher(r); + searcher.setSimilarity(sim); + Query query = new BoostQuery( + new BooleanQuery.Builder().add(new TermQuery(new Term("f", "foo")), Occur.SHOULD) + .add(new TermQuery(new Term("match", "yes")), Occur.FILTER) + .build(), + 3.2f + ); + TopDocs topDocs = searcher.search(query, 1); + assertEquals(1, topDocs.totalHits.value); + assertEquals((float) (3.2 * 2 / 3), topDocs.scoreDocs[0].score, 0); + } + } } } diff --git a/server/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java b/server/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java index bc8600a8cc05c..219242019cb45 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQueryTests.java @@ -25,31 +25,33 @@ public class MultiPhrasePrefixQueryTests extends ESTestCase { public void testSimple() throws Exception { - IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); - Document doc = new Document(); - doc.add(new Field("field", "aaa bbb ccc ddd", TextField.TYPE_NOT_STORED)); - writer.addDocument(doc); - IndexReader reader = DirectoryReader.open(writer); - IndexSearcher searcher = newSearcher(reader); - - MultiPhrasePrefixQuery query = new MultiPhrasePrefixQuery("field"); - query.add(new Term("field", "aa")); - assertThat(searcher.count(query), equalTo(1)); - - query = new MultiPhrasePrefixQuery("field"); - query.add(new Term("field", "aaa")); - query.add(new Term("field", "bb")); - assertThat(searcher.count(query), equalTo(1)); - - query = new MultiPhrasePrefixQuery("field"); - query.setSlop(1); - query.add(new Term("field", "aaa")); - query.add(new Term("field", "cc")); - assertThat(searcher.count(query), equalTo(1)); - - query = new MultiPhrasePrefixQuery("field"); - query.setSlop(1); - query.add(new Term("field", "xxx")); - assertThat(searcher.count(query), equalTo(0)); + try (IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER))) { + Document doc = new Document(); + doc.add(new Field("field", "aaa bbb ccc ddd", TextField.TYPE_NOT_STORED)); + writer.addDocument(doc); + try (IndexReader reader = DirectoryReader.open(writer)) { + IndexSearcher searcher = newSearcher(reader); + + MultiPhrasePrefixQuery query = new MultiPhrasePrefixQuery("field"); + query.add(new Term("field", "aa")); + assertThat(searcher.count(query), equalTo(1)); + + query = new MultiPhrasePrefixQuery("field"); + query.add(new Term("field", "aaa")); + query.add(new Term("field", "bb")); + assertThat(searcher.count(query), equalTo(1)); + + query = new MultiPhrasePrefixQuery("field"); + query.setSlop(1); + query.add(new Term("field", "aaa")); + query.add(new Term("field", "cc")); + assertThat(searcher.count(query), equalTo(1)); + + query = new MultiPhrasePrefixQuery("field"); + query.setSlop(1); + query.add(new Term("field", "xxx")); + assertThat(searcher.count(query), equalTo(0)); + } + } } } diff --git a/server/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java b/server/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java index 0fe25a6b74412..24e635b2f8b76 100644 --- a/server/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java +++ b/server/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java @@ -44,47 +44,55 @@ public class SimpleLuceneTests extends ESTestCase { public void testSortValues() throws Exception { - Directory dir = new ByteBuffersDirectory(); - IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); - for (int i = 0; i < 10; i++) { - Document document = new Document(); - String text = new String(new char[] { (char) (97 + i), (char) (97 + i) }); - document.add(new TextField("str", text, Field.Store.YES)); - document.add(new SortedDocValuesField("str", new BytesRef(text))); - indexWriter.addDocument(document); - } - IndexReader reader = DirectoryReader.open(indexWriter); - IndexSearcher searcher = newSearcher(reader); - TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("str", SortField.Type.STRING))); - for (int i = 0; i < 10; i++) { - FieldDoc fieldDoc = (FieldDoc) docs.scoreDocs[i]; - assertThat((BytesRef) fieldDoc.fields[0], equalTo(new BytesRef(new String(new char[] { (char) (97 + i), (char) (97 + i) })))); + try ( + Directory dir = new ByteBuffersDirectory(); + IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)) + ) { + for (int i = 0; i < 10; i++) { + Document document = new Document(); + String text = new String(new char[] { (char) (97 + i), (char) (97 + i) }); + document.add(new TextField("str", text, Field.Store.YES)); + document.add(new SortedDocValuesField("str", new BytesRef(text))); + indexWriter.addDocument(document); + } + try (IndexReader reader = DirectoryReader.open(indexWriter)) { + IndexSearcher searcher = newSearcher(reader); + TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("str", SortField.Type.STRING))); + for (int i = 0; i < 10; i++) { + FieldDoc fieldDoc = (FieldDoc) docs.scoreDocs[i]; + assertThat( + (BytesRef) fieldDoc.fields[0], + equalTo(new BytesRef(new String(new char[] { (char) (97 + i), (char) (97 + i) }))) + ); + } + } } } public void testSimpleNumericOps() throws Exception { - Directory dir = new ByteBuffersDirectory(); - IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); - - Document document = new Document(); - document.add(new TextField("_id", "1", Field.Store.YES)); - document.add(new IntPoint("test", 2)); - document.add(new StoredField("test", 2)); - indexWriter.addDocument(document); - - IndexReader reader = DirectoryReader.open(indexWriter); - IndexSearcher searcher = newSearcher(reader); - TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1); - Document doc = searcher.doc(topDocs.scoreDocs[0].doc); - IndexableField f = doc.getField("test"); - assertThat(f.numericValue(), equalTo(2)); - - topDocs = searcher.search(IntPoint.newExactQuery("test", 2), 1); - doc = searcher.doc(topDocs.scoreDocs[0].doc); - f = doc.getField("test"); - assertThat(f.stringValue(), equalTo("2")); + try ( + Directory dir = new ByteBuffersDirectory(); + IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)) + ) { + Document document = new Document(); + document.add(new TextField("_id", "1", Field.Store.YES)); + document.add(new IntPoint("test", 2)); + document.add(new StoredField("test", 2)); + indexWriter.addDocument(document); - indexWriter.close(); + try (IndexReader reader = DirectoryReader.open(indexWriter)) { + IndexSearcher searcher = newSearcher(reader); + TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1); + Document doc = searcher.doc(topDocs.scoreDocs[0].doc); + IndexableField f = doc.getField("test"); + assertThat(f.numericValue(), equalTo(2)); + + topDocs = searcher.search(IntPoint.newExactQuery("test", 2), 1); + doc = searcher.doc(topDocs.scoreDocs[0].doc); + f = doc.getField("test"); + assertThat(f.stringValue(), equalTo("2")); + } + } } /** @@ -93,54 +101,55 @@ public void testSimpleNumericOps() throws Exception { * first (with load and break). */ public void testOrdering() throws Exception { - Directory dir = new ByteBuffersDirectory(); - IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); - - Document document = new Document(); - document.add(new TextField("_id", "1", Field.Store.YES)); - document.add(new TextField("#id", "1", Field.Store.YES)); - indexWriter.addDocument(document); + try ( + Directory dir = new ByteBuffersDirectory(); + IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)) + ) { + Document document = new Document(); + document.add(new TextField("_id", "1", Field.Store.YES)); + document.add(new TextField("#id", "1", Field.Store.YES)); + indexWriter.addDocument(document); - IndexReader reader = DirectoryReader.open(indexWriter); - IndexSearcher searcher = newSearcher(reader); - TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1); - final ArrayList fieldsOrder = new ArrayList<>(); - searcher.doc(topDocs.scoreDocs[0].doc, new StoredFieldVisitor() { - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - fieldsOrder.add(fieldInfo.name); - return Status.YES; + try (IndexReader reader = DirectoryReader.open(indexWriter)) { + IndexSearcher searcher = newSearcher(reader); + TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1); + final ArrayList fieldsOrder = new ArrayList<>(); + searcher.doc(topDocs.scoreDocs[0].doc, new StoredFieldVisitor() { + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + fieldsOrder.add(fieldInfo.name); + return Status.YES; + } + }); + + assertThat(fieldsOrder.size(), equalTo(2)); + assertThat(fieldsOrder.get(0), equalTo("_id")); + assertThat(fieldsOrder.get(1), equalTo("#id")); } - }); - - assertThat(fieldsOrder.size(), equalTo(2)); - assertThat(fieldsOrder.get(0), equalTo("_id")); - assertThat(fieldsOrder.get(1), equalTo("#id")); - - indexWriter.close(); + } } public void testNRTSearchOnClosedWriter() throws Exception { - Directory dir = new ByteBuffersDirectory(); - IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); - DirectoryReader reader = DirectoryReader.open(indexWriter); - - for (int i = 0; i < 100; i++) { - Document document = new Document(); - TextField field = new TextField("_id", Integer.toString(i), Field.Store.YES); - document.add(field); - indexWriter.addDocument(document); - } - reader = refreshReader(reader); - - indexWriter.close(); - - for (LeafReaderContext leaf : reader.leaves()) { - leaf.reader().terms("_id").iterator().next(); + try ( + Directory dir = new ByteBuffersDirectory(); + IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); + DirectoryReader reader = DirectoryReader.open(indexWriter) + ) { + for (int i = 0; i < 100; i++) { + Document document = new Document(); + TextField field = new TextField("_id", Integer.toString(i), Field.Store.YES); + document.add(field); + indexWriter.addDocument(document); + } + try (DirectoryReader refreshedReader = refreshReader(reader)) { + for (LeafReaderContext leaf : refreshedReader.leaves()) { + leaf.reader().terms("_id").iterator().next(); + } + } } } - private DirectoryReader refreshReader(DirectoryReader reader) throws IOException { + private static DirectoryReader refreshReader(DirectoryReader reader) throws IOException { DirectoryReader oldReader = reader; reader = DirectoryReader.openIfChanged(reader); if (reader != oldReader) { From b08f1e4090f6dae3a582637caab9436fc471bcfb Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Fri, 8 Sep 2023 10:21:36 +0300 Subject: [PATCH 17/69] ESQL: "params" correctly parses the values including an optional "type" (#99310) --- docs/changelog/99310.yaml | 6 ++ .../resources/rest-api-spec/test/10_basic.yml | 26 ++++++ .../xpack/esql/qa/rest/RestEsqlTestCase.java | 59 ++++++++++++ .../xpack/esql/action/EsqlQueryRequest.java | 29 +++--- .../xpack/esql/parser/EsqlParser.java | 2 +- .../esql/action/EsqlQueryRequestTests.java | 91 ++++++++++++++----- 6 files changed, 172 insertions(+), 41 deletions(-) create mode 100644 docs/changelog/99310.yaml diff --git a/docs/changelog/99310.yaml b/docs/changelog/99310.yaml new file mode 100644 index 0000000000000..8b595fe93fd33 --- /dev/null +++ b/docs/changelog/99310.yaml @@ -0,0 +1,6 @@ +pr: 99310 +summary: "ESQL: \"params\" correctly parses the values including an optional \"type\"" +area: ES|QL +type: bug +issues: + - 99294 diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml index 9716004c3fbc1..fdd5cf2566961 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -297,3 +297,29 @@ setup: - match: {columns.2.type: "keyword"} - length: {values: 1} - match: {values.0: [1, 44, "green"]} + +--- +"Test Mixed Input Params": + - do: + esql.query: + body: + query: 'from test | eval x = ?, y = ?, z = ?, t = ?, u = ?, v = ? | keep x, y, z, t, u, v | limit 3' + params: [{"value": 1, "type": "keyword"}, {"value": 2, "type": "double"}, null, true, 123, {"value": 123, "type": "long"}] + + - length: {columns: 6} + - match: {columns.0.name: "x"} + - match: {columns.0.type: "keyword"} + - match: {columns.1.name: "y"} + - match: {columns.1.type: "double"} + - match: {columns.2.name: "z"} + - match: {columns.2.type: "null"} + - match: {columns.3.name: "t"} + - match: {columns.3.type: "boolean"} + - match: {columns.4.name: "u"} + - match: {columns.4.type: "integer"} + - match: {columns.5.name: "v"} + - match: {columns.5.type: "long"} + - length: {values: 3} + - match: {values.0: ["1",2.0,null,true,123,123]} + - match: {values.1: ["1",2.0,null,true,123,123]} + - match: {values.2: ["1",2.0,null,true,123,123]} diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 42bc9872c7e8e..7d90cf47cae09 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -16,6 +16,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; @@ -75,6 +76,11 @@ public RequestObjectBuilder columnar(boolean columnar) throws IOException { return this; } + public RequestObjectBuilder params(String rawParams) throws IOException { + builder.rawField("params", new BytesArray(rawParams).streamInput(), XContentType.JSON); + return this; + } + public RequestObjectBuilder timeZone(ZoneId zoneId) throws IOException { builder.field("time_zone", zoneId); return this; @@ -293,6 +299,59 @@ public void testMetadataFieldsOnMultipleIndices() throws IOException { assertMap(result, matchesMap().entry("columns", columns).entry("values", values)); } + public void testErrorMessageForEmptyParams() throws IOException { + ResponseException re = expectThrows( + ResponseException.class, + () -> runEsql(new RequestObjectBuilder().query("row a = 1 | eval x = ?").params("[]").build()) + ); + assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("Not enough actual parameters 0")); + } + + public void testErrorMessageForInvalidParams() throws IOException { + ResponseException re = expectThrows( + ResponseException.class, + () -> runEsql(new RequestObjectBuilder().query("row a = 1").params("[{\"x\":\"y\"}]").build()) + ); + assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("Required [value, type]")); + } + + public void testErrorMessageForMissingTypeInParams() throws IOException { + ResponseException re = expectThrows( + ResponseException.class, + () -> runEsql(new RequestObjectBuilder().query("row a = 1").params("[\"x\", 123, true, {\"value\": \"y\"}]").build()) + ); + assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("Required [type]")); + } + + public void testErrorMessageForMissingValueInParams() throws IOException { + ResponseException re = expectThrows( + ResponseException.class, + () -> runEsql(new RequestObjectBuilder().query("row a = 1").params("[\"x\", 123, true, {\"type\": \"y\"}]").build()) + ); + assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("Required [value]")); + } + + public void testErrorMessageForInvalidTypeInParams() throws IOException { + ResponseException re = expectThrows( + ResponseException.class, + () -> runEsql(new RequestObjectBuilder().query("row a = 1 | eval x = ?").params("[{\"type\": \"byte\", \"value\": 5}]").build()) + ); + assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("illegal data type [byte]")); + } + + public void testErrorMessageForArrayValuesInParams() throws IOException { + ResponseException re = expectThrows( + ResponseException.class, + () -> runEsql( + new RequestObjectBuilder().query("row a = 1 | eval x = ?").params("[{\"type\": \"integer\", \"value\": [5, 6, 7]}]").build() + ) + ); + assertThat( + EntityUtils.toString(re.getResponse().getEntity()), + containsString("[params] value doesn't support values of type: START_ARRAY") + ); + } + private static String expectedTextBody(String format, int count, @Nullable Character csvDelimiter) { StringBuilder sb = new StringBuilder(); switch (format) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java index 812b22a9857dc..5196cbb0dfd1c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java @@ -38,6 +38,8 @@ import java.util.function.Supplier; import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.common.xcontent.XContentParserUtils.parseFieldsValue; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ObjectParser.ValueType.VALUE_ARRAY; public class EsqlQueryRequest extends ActionRequest implements CompositeIndicesRequest { @@ -47,6 +49,13 @@ public class EsqlQueryRequest extends ActionRequest implements CompositeIndicesR true, objects -> new TypedParamValue((String) objects[1], objects[0]) ); + private static final ParseField VALUE = new ParseField("value"); + private static final ParseField TYPE = new ParseField("type"); + + static { + PARAM_PARSER.declareField(constructorArg(), (p, c) -> parseFieldsValue(p), VALUE, ObjectParser.ValueType.VALUE); + PARAM_PARSER.declareString(constructorArg(), TYPE); + } private static final ParseField QUERY_FIELD = new ParseField("query"); private static final ParseField COLUMNAR_FIELD = new ParseField("columnar"); @@ -141,9 +150,7 @@ public void params(List params) { } public static EsqlQueryRequest fromXContent(XContentParser parser) { - EsqlQueryRequest result = PARSER.apply(parser, null); - validateParams(result.params); - return result; + return PARSER.apply(parser, null); } private static ObjectParser objectParser(Supplier supplier) { @@ -171,7 +178,7 @@ private static List parseParams(XContentParser p) throws IOExce Object value = null; String type = null; TypedParamValue previousParam = null; - TypedParamValue currentParam = null; + TypedParamValue currentParam; while ((token = p.nextToken()) != XContentParser.Token.END_ARRAY) { XContentLocation loc = p.getTokenLocation(); @@ -200,9 +207,6 @@ private static List parseParams(XContentParser p) throws IOExce } else if (numberType == XContentParser.NumberType.LONG) { value = p.longValue(); type = "long"; - } else if (numberType == XContentParser.NumberType.FLOAT) { - value = p.floatValue(); - type = "float"; } else if (numberType == XContentParser.NumberType.DOUBLE) { value = p.doubleValue(); type = "double"; @@ -244,17 +248,6 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, return new CancellableTask(id, type, action, query, parentTaskId, headers); } - protected static void validateParams(List params) { - for (TypedParamValue param : params) { - if (param.hasExplicitType()) { - throw new XContentParseException( - fromProto(param.tokenLocation()), - "[params] must be an array where each entry is a single field (no " + "objects supported)" - ); - } - } - } - static org.elasticsearch.xcontent.XContentLocation fromProto(ContentLocation fromProto) { if (fromProto == null) { return null; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java index b6f73e9ee7722..2f527c54430a7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java @@ -135,7 +135,7 @@ public Token nextToken() { Token token = delegate.nextToken(); if (token.getType() == EsqlBaseLexer.PARAM) { if (param >= params.size()) { - throw new ParsingException("Not enough actual parameters {} ", params.size()); + throw new ParsingException("Not enough actual parameters {}", params.size()); } paramTokens.put(token, params.get(param)); param++; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java index 9fb12572fd7ce..dd25148c958d0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java @@ -23,13 +23,16 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.esql.parser.TypedParamValue; import java.io.IOException; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.Supplier; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -42,33 +45,18 @@ public void testParseFields() throws IOException { ZoneId zoneId = randomZone(); Locale locale = randomLocale(random()); QueryBuilder filter = randomQueryBuilder(); - List params = randomList(5, () -> randomBoolean() ? randomInt(100) : randomAlphaOfLength(10)); - StringBuilder paramsString = new StringBuilder(); - paramsString.append("["); - boolean first = true; - for (Object param : params) { - if (first == false) { - paramsString.append(", "); - } - first = false; - if (param instanceof String) { - paramsString.append("\""); - paramsString.append(param); - paramsString.append("\""); - } else { - paramsString.append(param); - } - } - paramsString.append("]"); + + List params = randomParameters(); + boolean hasParams = params.isEmpty() == false; + StringBuilder paramsString = paramsString(params, hasParams); String json = String.format(Locale.ROOT, """ { "query": "%s", "columnar": %s, "time_zone": "%s", "locale": "%s", - "filter": %s, - "params": %s - }""", query, columnar, zoneId, locale.toLanguageTag(), filter, paramsString); + "filter": %s + %s""", query, columnar, zoneId, locale.toLanguageTag(), filter, paramsString); EsqlQueryRequest request = parseEsqlQueryRequest(json); @@ -81,7 +69,7 @@ public void testParseFields() throws IOException { assertEquals(params.size(), request.params().size()); for (int i = 0; i < params.size(); i++) { - assertEquals(params.get(i), request.params().get(i).value); + assertEquals(params.get(i), request.params().get(i)); } } @@ -136,6 +124,65 @@ public void testTask() throws IOException { assertThat(json, equalTo(expected)); } + private List randomParameters() { + if (randomBoolean()) { + return Collections.emptyList(); + } else { + int len = randomIntBetween(1, 10); + List arr = new ArrayList<>(len); + for (int i = 0; i < len; i++) { + boolean hasExplicitType = randomBoolean(); + @SuppressWarnings("unchecked") + Supplier supplier = randomFrom( + () -> new TypedParamValue("boolean", randomBoolean(), hasExplicitType), + () -> new TypedParamValue("integer", randomInt(), hasExplicitType), + () -> new TypedParamValue("long", randomLong(), hasExplicitType), + () -> new TypedParamValue("double", randomDouble(), hasExplicitType), + () -> new TypedParamValue("null", null, hasExplicitType), + () -> new TypedParamValue("keyword", randomAlphaOfLength(10), hasExplicitType) + ); + arr.add(supplier.get()); + } + return Collections.unmodifiableList(arr); + } + } + + private StringBuilder paramsString(List params, boolean hasParams) { + StringBuilder paramsString = new StringBuilder(); + if (hasParams) { + paramsString.append(",\"params\":["); + boolean first = true; + for (TypedParamValue param : params) { + if (first == false) { + paramsString.append(", "); + } + first = false; + if (param.hasExplicitType()) { + paramsString.append("{\"type\":\""); + paramsString.append(param.type); + paramsString.append("\",\"value\":"); + } + switch (param.type) { + case "keyword" -> { + paramsString.append("\""); + paramsString.append(param.value); + paramsString.append("\""); + } + case "integer", "long", "boolean", "null", "double" -> { + paramsString.append(param.value); + } + } + if (param.hasExplicitType()) { + paramsString.append("}"); + } + } + paramsString.append("]}"); + } else { + paramsString.append("}"); + } + return paramsString; + } + private static void assertParserErrorMessage(String json, String message) { Exception e = expectThrows(IllegalArgumentException.class, () -> parseEsqlQueryRequest(json)); assertThat(e.getMessage(), containsString(message)); From 6b85f740cdf00c83e3c2e912455a4aecd7a5e4ff Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 8 Sep 2023 08:38:15 +0100 Subject: [PATCH 18/69] Migrate AbstractAsyncTask away from name-based executor (#99337) Replaces the executor name returned from a protected method with a proper executor passed into the constructor, and removes the default implementation that dangerously defaults to running the task on the scheduler thread so that callers have to make a conscious choice of executor. Relates #99027 and friends --- .../util/concurrent/AbstractAsyncTask.java | 15 ++--- .../org/elasticsearch/index/IndexService.java | 55 ++++++++----------- .../PersistentTasksClusterService.java | 4 +- .../concurrent/AbstractAsyncTaskTests.java | 37 ++++++++----- .../index/IndexServiceTests.java | 25 ++++----- .../shared/SharedBlobCacheService.java | 11 +--- .../operator/exchange/ExchangeService.java | 3 +- .../cache/full/CacheService.java | 7 +-- 8 files changed, 72 insertions(+), 85 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java index 031ff232351a8..add11b03c5f1c 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTask.java @@ -14,6 +14,7 @@ import java.io.Closeable; import java.util.Objects; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.core.Strings.format; @@ -25,6 +26,7 @@ public abstract class AbstractAsyncTask implements Runnable, Closeable { private final Logger logger; private final ThreadPool threadPool; + private final Executor executor; private final AtomicBoolean closed = new AtomicBoolean(false); private final boolean autoReschedule; private volatile Scheduler.Cancellable cancellable; @@ -32,9 +34,10 @@ public abstract class AbstractAsyncTask implements Runnable, Closeable { private volatile Exception lastThrownException; private volatile TimeValue interval; - protected AbstractAsyncTask(Logger logger, ThreadPool threadPool, TimeValue interval, boolean autoReschedule) { + protected AbstractAsyncTask(Logger logger, ThreadPool threadPool, Executor executor, TimeValue interval, boolean autoReschedule) { this.logger = logger; this.threadPool = threadPool; + this.executor = executor; this.interval = interval; this.autoReschedule = autoReschedule; } @@ -81,7 +84,7 @@ public synchronized void rescheduleIfNecessary() { if (logger.isTraceEnabled()) { logger.trace("scheduling {} every {}", toString(), interval); } - cancellable = threadPool.schedule(this, interval, getThreadPool()); + cancellable = threadPool.schedule(this, interval, executor); isScheduledOrRunning = true; } else { logger.trace("scheduled {} disabled", toString()); @@ -167,12 +170,4 @@ private static boolean sameException(Exception left, Exception right) { } protected abstract void runInternal(); - - /** - * Use the same threadpool by default. - * Derived classes can change this if required. - */ - protected String getThreadPool() { - return ThreadPool.Names.SAME; - } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index ded7e459403eb..05c6fd63c3fcb 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -98,6 +98,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BooleanSupplier; @@ -1062,8 +1063,8 @@ abstract static class BaseAsyncTask extends AbstractAsyncTask { protected final IndexService indexService; - BaseAsyncTask(final IndexService indexService, final TimeValue interval) { - super(indexService.logger, indexService.threadPool, interval, true); + BaseAsyncTask(final IndexService indexService, final Executor executor, final TimeValue interval) { + super(indexService.logger, indexService.threadPool, executor, interval, true); this.indexService = indexService; rescheduleIfNecessary(); } @@ -1082,12 +1083,11 @@ protected boolean mustReschedule() { static final class AsyncTranslogFSync extends BaseAsyncTask { AsyncTranslogFSync(IndexService indexService) { - super(indexService, indexService.getIndexSettings().getTranslogSyncInterval()); - } - - @Override - protected String getThreadPool() { - return ThreadPool.Names.FLUSH; + super( + indexService, + indexService.threadPool.executor(ThreadPool.Names.FLUSH), + indexService.getIndexSettings().getTranslogSyncInterval() + ); } @Override @@ -1111,7 +1111,11 @@ public String toString() { static final class AsyncRefreshTask extends BaseAsyncTask { AsyncRefreshTask(IndexService indexService) { - super(indexService, indexService.getIndexSettings().getRefreshInterval()); + super( + indexService, + indexService.threadPool.executor(ThreadPool.Names.REFRESH), + indexService.getIndexSettings().getRefreshInterval() + ); } @Override @@ -1119,11 +1123,6 @@ protected void runInternal() { indexService.maybeRefreshEngine(false); } - @Override - protected String getThreadPool() { - return ThreadPool.Names.REFRESH; - } - @Override public String toString() { return "refresh"; @@ -1135,6 +1134,7 @@ final class AsyncTrimTranslogTask extends BaseAsyncTask { AsyncTrimTranslogTask(IndexService indexService) { super( indexService, + threadPool.generic(), indexService.getIndexSettings() .getSettings() .getAsTime(INDEX_TRANSLOG_RETENTION_CHECK_INTERVAL_SETTING, TimeValue.timeValueMinutes(10)) @@ -1151,11 +1151,6 @@ protected void runInternal() { indexService.maybeTrimTranslog(); } - @Override - protected String getThreadPool() { - return ThreadPool.Names.GENERIC; - } - @Override public String toString() { return "trim_translog"; @@ -1187,7 +1182,11 @@ private static final class AsyncGlobalCheckpointTask extends BaseAsyncTask { AsyncGlobalCheckpointTask(final IndexService indexService) { // index.global_checkpoint_sync_interval is not a real setting, it is only registered in tests - super(indexService, GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.get(indexService.getIndexSettings().getSettings())); + super( + indexService, + indexService.getThreadPool().generic(), + GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.get(indexService.getIndexSettings().getSettings()) + ); } @Override @@ -1195,11 +1194,6 @@ protected void runInternal() { indexService.maybeSyncGlobalCheckpoints(); } - @Override - protected String getThreadPool() { - return ThreadPool.Names.GENERIC; - } - @Override public String toString() { return "global_checkpoint_sync"; @@ -1209,7 +1203,11 @@ public String toString() { private static final class AsyncRetentionLeaseSyncTask extends BaseAsyncTask { AsyncRetentionLeaseSyncTask(final IndexService indexService) { - super(indexService, RETENTION_LEASE_SYNC_INTERVAL_SETTING.get(indexService.getIndexSettings().getSettings())); + super( + indexService, + indexService.threadPool.executor(ThreadPool.Names.MANAGEMENT), + RETENTION_LEASE_SYNC_INTERVAL_SETTING.get(indexService.getIndexSettings().getSettings()) + ); } @Override @@ -1217,11 +1215,6 @@ protected void runInternal() { indexService.syncRetentionLeases(); } - @Override - protected String getThreadPool() { - return ThreadPool.Names.MANAGEMENT; - } - @Override public String toString() { return "retention_lease_sync"; diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java index e2ca66f3835a9..e2d7d0c8366d8 100644 --- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java +++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractAsyncTask; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; @@ -524,7 +525,7 @@ private static Assignment unassignedAssignment(String reason) { class PeriodicRechecker extends AbstractAsyncTask { PeriodicRechecker(TimeValue recheckInterval) { - super(logger, threadPool, recheckInterval, false); + super(logger, threadPool, EsExecutors.DIRECT_EXECUTOR_SERVICE, recheckInterval, false); } @Override @@ -535,6 +536,7 @@ protected boolean mustReschedule() { @Override public void runInternal() { if (clusterService.localNode().isMasterNode()) { + // TODO just run on the elected master? final ClusterState state = clusterService.state(); logger.trace("periodic persistent task assignment check running for cluster state {}", state.getVersion()); if (isAnyTaskUnassigned(state.getMetadata().custom(PersistentTasksCustomMetadata.TYPE))) { diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTaskTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTaskTests.java index 0a7bf591bb2c7..cdb85aaae2fc6 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTaskTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/AbstractAsyncTaskTests.java @@ -45,7 +45,7 @@ public void testAutoRepeat() throws Exception { final CyclicBarrier barrier1 = new CyclicBarrier(2); // 1 for runInternal plus 1 for the test sequence final CyclicBarrier barrier2 = new CyclicBarrier(2); // 1 for runInternal plus 1 for the test sequence final AtomicInteger count = new AtomicInteger(); - AbstractAsyncTask task = new AbstractAsyncTask(logger, threadPool, TimeValue.timeValueMillis(1), true) { + AbstractAsyncTask task = new AbstractAsyncTask(logger, threadPool, threadPool.generic(), TimeValue.timeValueMillis(1), true) { @Override protected boolean mustReschedule() { @@ -71,10 +71,6 @@ protected void runInternal() { } } - @Override - protected String getThreadPool() { - return ThreadPool.Names.GENERIC; - } }; assertFalse(task.isScheduled()); @@ -101,7 +97,7 @@ public void testManualRepeat() throws Exception { boolean shouldRunThrowException = randomBoolean(); final CyclicBarrier barrier = new CyclicBarrier(2); // 1 for runInternal plus 1 for the test sequence final AtomicInteger count = new AtomicInteger(); - AbstractAsyncTask task = new AbstractAsyncTask(logger, threadPool, TimeValue.timeValueMillis(1), false) { + AbstractAsyncTask task = new AbstractAsyncTask(logger, threadPool, threadPool.generic(), TimeValue.timeValueMillis(1), false) { @Override protected boolean mustReschedule() { @@ -122,10 +118,6 @@ protected void runInternal() { } } - @Override - protected String getThreadPool() { - return ThreadPool.Names.GENERIC; - } }; assertFalse(task.isScheduled()); @@ -148,7 +140,13 @@ protected String getThreadPool() { public void testCloseWithNoRun() { - AbstractAsyncTask task = new AbstractAsyncTask(logger, threadPool, TimeValue.timeValueMinutes(10), true) { + AbstractAsyncTask task = new AbstractAsyncTask( + logger, + threadPool, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TimeValue.timeValueMinutes(10), + true + ) { @Override protected boolean mustReschedule() { @@ -171,7 +169,13 @@ public void testChangeInterval() throws Exception { final CountDownLatch latch = new CountDownLatch(2); - AbstractAsyncTask task = new AbstractAsyncTask(logger, threadPool, TimeValue.timeValueHours(1), true) { + AbstractAsyncTask task = new AbstractAsyncTask( + logger, + threadPool, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TimeValue.timeValueHours(1), + true + ) { @Override protected boolean mustReschedule() { @@ -202,7 +206,14 @@ public void testIsScheduledRemainFalseAfterClose() throws Exception { List tasks = new ArrayList<>(numTasks); AtomicLong counter = new AtomicLong(); for (int i = 0; i < numTasks; i++) { - AbstractAsyncTask task = new AbstractAsyncTask(logger, threadPool, TimeValue.timeValueMillis(randomIntBetween(1, 2)), true) { + AbstractAsyncTask task = new AbstractAsyncTask( + logger, + threadPool, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TimeValue.timeValueMillis(randomIntBetween(1, 2)), + true + ) { + @Override protected boolean mustReschedule() { return counter.get() <= 1000; diff --git a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java index dba69e7db7ba4..eb034778be63d 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.query.QueryBuilder; @@ -24,7 +25,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -75,7 +75,11 @@ public void testBaseAsyncTask() throws Exception { AtomicReference latch = new AtomicReference<>(new CountDownLatch(1)); AtomicReference latch2 = new AtomicReference<>(new CountDownLatch(1)); final AtomicInteger count = new AtomicInteger(); - IndexService.BaseAsyncTask task = new IndexService.BaseAsyncTask(indexService, TimeValue.timeValueMillis(1)) { + IndexService.BaseAsyncTask task = new IndexService.BaseAsyncTask( + indexService, + indexService.getThreadPool().generic(), + TimeValue.timeValueMillis(1) + ) { @Override protected void runInternal() { final CountDownLatch l1 = latch.get(); @@ -96,11 +100,6 @@ protected void runInternal() { } } } - - @Override - protected String getThreadPool() { - return ThreadPool.Names.GENERIC; - } }; latch.get().await(); @@ -115,11 +114,9 @@ protected String getThreadPool() { latch2.get().countDown(); assertEquals(2, count.get()); - task = new IndexService.BaseAsyncTask(indexService, TimeValue.timeValueMillis(1000000)) { + task = new IndexService.BaseAsyncTask(indexService, EsExecutors.DIRECT_EXECUTOR_SERVICE, TimeValue.timeValueMillis(1000000)) { @Override - protected void runInternal() { - - } + protected void runInternal() {} }; assertTrue(task.mustReschedule()); @@ -140,11 +137,9 @@ protected void runInternal() { indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(index); assertNotSame(closedIndexService, indexService); - task = new IndexService.BaseAsyncTask(indexService, TimeValue.timeValueMillis(100000)) { + task = new IndexService.BaseAsyncTask(indexService, EsExecutors.DIRECT_EXECUTOR_SERVICE, TimeValue.timeValueMillis(100000)) { @Override - protected void runInternal() { - - } + protected void runInternal() {} }; assertTrue(task.mustReschedule()); assertFalse(task.isClosed()); diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java index 493d25d685768..d7cbddf490df5 100644 --- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java +++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java @@ -336,7 +336,7 @@ public SharedBlobCacheService( for (int i = 0; i < numRegions; i++) { freeRegions.add(sharedBytes.getFileChannel(i)); } - decayTask = new CacheDecayTask(threadPool, SHARED_CACHE_DECAY_INTERVAL_SETTING.get(settings)); + decayTask = new CacheDecayTask(threadPool, threadPool.generic(), SHARED_CACHE_DECAY_INTERVAL_SETTING.get(settings)); decayTask.rescheduleIfNecessary(); this.rangeSize = SHARED_CACHE_RANGE_SIZE_SETTING.get(settings); this.recoveryRangeSize = SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING.get(settings); @@ -745,8 +745,8 @@ public void close() { class CacheDecayTask extends AbstractAsyncTask { - CacheDecayTask(ThreadPool threadPool, TimeValue interval) { - super(logger, Objects.requireNonNull(threadPool), Objects.requireNonNull(interval), true); + CacheDecayTask(ThreadPool threadPool, Executor executor, TimeValue interval) { + super(logger, Objects.requireNonNull(threadPool), executor, Objects.requireNonNull(interval), true); } @Override @@ -759,11 +759,6 @@ public void runInternal() { computeDecay(); } - @Override - protected String getThreadPool() { - return ThreadPool.Names.GENERIC; - } - @Override public String toString() { return "shared_cache_decay_task"; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java index 1054e240eed6b..c3efd67579d66 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractAsyncTask; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; @@ -211,7 +212,7 @@ public void messageReceived(ExchangeRequest request, TransportChannel channel, T private final class InactiveSinksReaper extends AbstractAsyncTask { InactiveSinksReaper(Logger logger, ThreadPool threadPool, TimeValue interval) { - super(logger, threadPool, interval, true); + super(logger, threadPool, EsExecutors.DIRECT_EXECUTOR_SERVICE, interval, true); rescheduleIfNecessary(); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java index 3daa72efdd608..db56addb434c7 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java @@ -636,7 +636,7 @@ public void synchronizeCache() { class CacheSynchronizationTask extends AbstractAsyncTask { CacheSynchronizationTask(ThreadPool threadPool, TimeValue interval) { - super(logger, Objects.requireNonNull(threadPool), Objects.requireNonNull(interval), true); + super(logger, Objects.requireNonNull(threadPool), threadPool.generic(), Objects.requireNonNull(interval), true); } @Override @@ -649,11 +649,6 @@ public void runInternal() { synchronizeCache(); } - @Override - protected String getThreadPool() { - return ThreadPool.Names.GENERIC; - } - @Override public String toString() { return "cache_synchronization_task"; From a40b61a0d8b2a5fc756f039e8672640d475365b1 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 8 Sep 2023 08:58:23 +0100 Subject: [PATCH 19/69] Mute YAML test for #99318 --- .../rest-api-spec/test/search.vectors/90_sparse_vector.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml index f0bb61c4c21af..eb967d6d662c0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml @@ -122,8 +122,10 @@ --- "Sparse vector in 8.x": - skip: - version: " - 7.99.99, 8.11.0 - " - reason: "sparse_vector field type not supported in 8.x until 8.11.0" + # version: " - 7.99.99, 8.11.0 - " + # reason: "sparse_vector field type not supported in 8.x until 8.11.0" + version: "all" + reason: "https://github.com/elastic/elasticsearch/issues/99318" - do: catch: /The \[sparse_vector\] field type is no longer supported/ indices.create: From 61228b888ea657b83cbf1803a25fee5ec0cd3e63 Mon Sep 17 00:00:00 2001 From: Stef Nestor <26751266+stefnestor@users.noreply.github.com> Date: Fri, 8 Sep 2023 02:28:30 -0600 Subject: [PATCH 20/69] [DOC+] ILM Searchable Snapshot migrations require repository "name" to be the same (#99308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 👋 howdy, team! Expanding reference to [internal](https://github.com/elastic/cloud/pull/118105) update, we've just confirmed ILM requires the repository name to be the same among migrating clusters. This is a hard block for Searchable Snapshots which requires un-Searchable-Snapshotting or redoing migration to resolve. --- docs/reference/snapshot-restore/apis/put-repo-api.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc index 4d578b3df489d..a50d4e3311937 100644 --- a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc @@ -17,6 +17,8 @@ PUT /_snapshot/my_repository } ---- +IMPORTANT: If you're migrating {ref}/searchable-snapshots.html[searchable snapshots], the repository's name must be identical in the source and destination clusters. + [[put-snapshot-repo-api-request]] ==== {api-request-title} From 216ef1719855c4cba2573eef6e4b8059111cef6c Mon Sep 17 00:00:00 2001 From: Pooya Salehi Date: Fri, 8 Sep 2023 11:03:33 +0200 Subject: [PATCH 21/69] Add yaml test debugging tip (#99341) --- TESTING.asciidoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/TESTING.asciidoc b/TESTING.asciidoc index 305e0f6371833..d33121a15dcf7 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -453,6 +453,11 @@ You can run a group of YAML test by using wildcards: --tests "org.elasticsearch.test.rest.ClientYamlTestSuiteIT.test {yaml=index/*/*}" --------------------------------------------------------------------------- +Note that if the selected test via the `--tests` filter is not a valid test, i.e., the YAML test +runner is not able to parse and load it, you might get an error message indicating that the test +was not found. In such cases, running the whole suite without using the `--tests` could show more +specific error messages about why the test runner is not able to parse or load a certain test. + The YAML REST tests support all the options provided by the randomized runner, plus the following: * `tests.rest.blacklist`: comma separated globs that identify tests that are From d507295cb2deaa9f5db8e26064b419c9ed923a65 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Fri, 8 Sep 2023 11:07:05 +0200 Subject: [PATCH 22/69] ES|QL: Add multi-node Spec tests (#98849) --- .../esql/qa/server/multi-node/build.gradle | 19 +++++++++++++++++++ .../xpack/esql/qa/multi_node/EsqlSpecIT.java | 17 +++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 x-pack/plugin/esql/qa/server/multi-node/build.gradle create mode 100644 x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java diff --git a/x-pack/plugin/esql/qa/server/multi-node/build.gradle b/x-pack/plugin/esql/qa/server/multi-node/build.gradle new file mode 100644 index 0000000000000..1b62fdea2671c --- /dev/null +++ b/x-pack/plugin/esql/qa/server/multi-node/build.gradle @@ -0,0 +1,19 @@ +apply plugin: 'elasticsearch.legacy-yaml-rest-test' + +dependencies { + javaRestTestImplementation project(xpackModule('esql:qa:testFixtures')) +} + +restResources { + restApi { + include '_common', 'bulk', 'indices', 'esql', 'xpack', 'enrich' + } +} + +testClusters.configureEach { + numberOfNodes = 2 + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.monitoring.collection.enabled', 'true' + setting 'xpack.security.enabled', 'false' +} diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java new file mode 100644 index 0000000000000..eab26b565f93d --- /dev/null +++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.qa.multi_node; + +import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase; +import org.elasticsearch.xpack.ql.CsvSpecReader.CsvTestCase; + +public class EsqlSpecIT extends EsqlSpecTestCase { + public EsqlSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) { + super(fileName, groupName, testName, lineNumber, testCase); + } +} From 3e3ee42589ea16cb44bb4eb4be7b3e5189336da3 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 8 Sep 2023 11:11:43 +0200 Subject: [PATCH 23/69] Add index.look_back_time setting for tsdb data streams (#98518) This change adds a `index.look_back_time` index setting that sets the `index.time_series.start_time` setting for the first backing index when a data stream is created. This allows accepting data that is older for initial indexing without changing the `index.look_ahead_time` setting. This setting also controls the `index.time_series.end_time` setting and would affect rollovers as well. The default for the `index.look_back_time` is `2h`, which means documents with `@timestamp` up to 2 hours after creation of the data stream are allowed to be indexed. This is the same as is without this change, because `index.look_ahead_time` is used to set `index.time_series.start_time` of the first backing index. Closes #98463 --- docs/changelog/98518.yaml | 6 ++ .../data-streams/set-up-tsds.asciidoc | 1 + .../data-streams/tsds-index-settings.asciidoc | 9 +++ docs/reference/data-streams/tsds.asciidoc | 16 +++++ .../datastreams/TsdbDataStreamRestIT.java | 65 +++++++++++++++++++ .../DataStreamIndexSettingsProvider.java | 3 +- .../datastreams/DataStreamsPlugin.java | 10 ++- .../DataStreamIndexSettingsProviderTests.java | 28 +++++++- .../datastreams/LookAHeadTimeTests.java | 19 ++++-- 9 files changed, 148 insertions(+), 9 deletions(-) create mode 100644 docs/changelog/98518.yaml diff --git a/docs/changelog/98518.yaml b/docs/changelog/98518.yaml new file mode 100644 index 0000000000000..2f961fc11ce69 --- /dev/null +++ b/docs/changelog/98518.yaml @@ -0,0 +1,6 @@ +pr: 98518 +summary: Add `index.look_back_time` setting for tsdb data streams +area: TSDB +type: enhancement +issues: + - 98463 diff --git a/docs/reference/data-streams/set-up-tsds.asciidoc b/docs/reference/data-streams/set-up-tsds.asciidoc index 3c15011871f89..a98e3c7302424 100644 --- a/docs/reference/data-streams/set-up-tsds.asciidoc +++ b/docs/reference/data-streams/set-up-tsds.asciidoc @@ -177,6 +177,7 @@ Optionally, the index settings component template for a TSDS can include: * Your lifecycle policy in the `index.lifecycle.name` index setting. * The <> index setting. +* The <> index setting. * Other index settings, such as <>, for your TSDS's backing indices. diff --git a/docs/reference/data-streams/tsds-index-settings.asciidoc b/docs/reference/data-streams/tsds-index-settings.asciidoc index fa5c9b8cd821f..8091163ffe883 100644 --- a/docs/reference/data-streams/tsds-index-settings.asciidoc +++ b/docs/reference/data-streams/tsds-index-settings.asciidoc @@ -33,6 +33,15 @@ days). Only indices with an `index.mode` of `time_series` support this setting. For more information, refer to <>. Additionally this setting can not be less than `time_series.poll_interval` cluster setting. +[[index-look-back-time]] +`index.look_back_time`:: +(<<_static_index_settings,Static>>, <>) +Interval used to calculate the `index.time_series.start_time` for a TSDS's first +backing index when a tsdb data stream is created. Defaults to `2h` (2 hours). +Accepts `1m` (one minute) to `7d` (seven days). Only indices with an `index.mode` +of `time_series` support this setting. For more information, +refer to <>. + [[index-routing-path]] `index.routing_path`:: (<<_static_index_settings,Static>>, string or array of strings) Plain `keyword` fields used to route documents in a TSDS to index shards. Supports wildcards diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc index 3f49a7ab8c700..d6e9ea08f0892 100644 --- a/docs/reference/data-streams/tsds.asciidoc +++ b/docs/reference/data-streams/tsds.asciidoc @@ -253,6 +253,22 @@ value borders the `index.time_series.start_time` for the new write index. This ensures the `@timestamp` ranges for neighboring backing indices always border but never overlap. +[discrete] +[[tsds-look-back-time]] +==== Look-back time + +Use the <> index setting to +configure how far in the past you can add documents to an index. When you +create a data stream for a TSDS, {es} calculates the index's +`index.time_series.start_time` value as: + +`now - index.look_back_time` + +This setting is only used when a data stream gets created and controls +the `index.time_series.start_time` index setting of the first backing index. +Configuring this index setting can be useful to accept documents with `@timestamp` +field values that are older than 2 hours (the `index.look_back_time` default). + [discrete] [[tsds-accepted-time-range]] ==== Accepted time range for adding data diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index 1f27c43f2f2f4..2a4b6f0c5a5ee 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -10,7 +10,9 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.test.rest.ObjectPath; import org.junit.Before; @@ -618,6 +620,69 @@ public void testUpdateComponentTemplateDoesNotFailIndexTemplateValidation() thro client().performRequest(request); } + public void testLookBackTime() throws IOException { + // Create template that uses index.look_back_time index setting: + String template = """ + { + "index_patterns": ["test*"], + "template": { + "settings":{ + "index": { + "look_back_time": "24h", + "number_of_replicas": 0, + "mode": "time_series" + } + }, + "mappings":{ + "properties": { + "@timestamp" : { + "type": "date" + }, + "field": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }, + "data_stream": {} + }"""; + var putIndexTemplateRequest = new Request("PUT", "/_index_template/2"); + putIndexTemplateRequest.setJsonEntity(template); + assertOK(client().performRequest(putIndexTemplateRequest)); + + // Create data stream: + var createDataStreamRequest = new Request("PUT", "/_data_stream/test123"); + assertOK(client().performRequest(createDataStreamRequest)); + + // Check data stream has been created: + var getDataStreamsRequest = new Request("GET", "/_data_stream"); + var response = client().performRequest(getDataStreamsRequest); + assertOK(response); + var dataStreams = entityAsMap(response); + assertThat(ObjectPath.evaluate(dataStreams, "data_streams"), hasSize(1)); + assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.name"), equalTo("test123")); + assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.generation"), equalTo(1)); + assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.template"), equalTo("2")); + assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.indices"), hasSize(1)); + String firstBackingIndex = ObjectPath.evaluate(dataStreams, "data_streams.0.indices.0.index_name"); + assertThat(firstBackingIndex, backingIndexEqualTo("test123", 1)); + + // Check the backing index: + // 2023-08-15T04:35:50.000Z + var indices = getIndex(firstBackingIndex); + var escapedBackingIndex = firstBackingIndex.replace(".", "\\."); + assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".data_stream"), equalTo("test123")); + assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.mode"), equalTo("time_series")); + String startTimeFirstBackingIndex = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.start_time"); + assertThat(startTimeFirstBackingIndex, notNullValue()); + Instant now = Instant.now(); + Instant startTime = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(startTimeFirstBackingIndex)).toInstant(); + assertTrue(now.minus(24, ChronoUnit.HOURS).isAfter(startTime)); + String endTimeFirstBackingIndex = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.end_time"); + assertThat(endTimeFirstBackingIndex, notNullValue()); + } + private static Map getIndex(String indexName) throws IOException { var getIndexRequest = new Request("GET", "/" + indexName + "?human"); var response = client().performRequest(getIndexRequest); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index f4e660c2b18f4..064030ed2b6d5 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -84,10 +84,11 @@ public Settings getAdditionalIndexSettings( if (indexMode == IndexMode.TIME_SERIES) { Settings.Builder builder = Settings.builder(); TimeValue lookAheadTime = DataStreamsPlugin.LOOK_AHEAD_TIME.get(allSettings); + TimeValue lookBackTime = DataStreamsPlugin.LOOK_BACK_TIME.get(allSettings); final Instant start; final Instant end; if (dataStream == null || migrating) { - start = DataStream.getCanonicalTimestampBound(resolvedAt.minusMillis(lookAheadTime.getMillis())); + start = DataStream.getCanonicalTimestampBound(resolvedAt.minusMillis(lookBackTime.getMillis())); end = DataStream.getCanonicalTimestampBound(resolvedAt.plusMillis(lookAheadTime.getMillis())); } else { IndexMetadata currentLatestBackingIndex = metadata.index(dataStream.getWriteIndex()); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index 3f8b7e40eeb43..cd221ada7a4dc 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -105,7 +105,14 @@ public class DataStreamsPlugin extends Plugin implements ActionPlugin { Setting.Property.Dynamic ); public static final String LIFECYCLE_CUSTOM_INDEX_METADATA_KEY = "data_stream_lifecycle"; - + public static final Setting LOOK_BACK_TIME = Setting.timeSetting( + "index.look_back_time", + TimeValue.timeValueHours(2), + TimeValue.timeValueMinutes(1), + TimeValue.timeValueDays(7), + Setting.Property.IndexScope, + Setting.Property.Dynamic + ); // The dependency of index.look_ahead_time is a cluster setting and currently there is no clean validation approach for this: private final SetOnce updateTimeSeriesRangeService = new SetOnce<>(); private final SetOnce errorStoreInitialisationService = new SetOnce<>(); @@ -141,6 +148,7 @@ public List> getSettings() { List> pluginSettings = new ArrayList<>(); pluginSettings.add(TIME_SERIES_POLL_INTERVAL); pluginSettings.add(LOOK_AHEAD_TIME); + pluginSettings.add(LOOK_BACK_TIME); pluginSettings.add(DataStreamLifecycleService.DATA_STREAM_LIFECYCLE_POLL_INTERVAL_SETTING); pluginSettings.add(DataStreamLifecycleService.DATA_STREAM_MERGE_POLICY_TARGET_FLOOR_SEGMENT_SETTING); pluginSettings.add(DataStreamLifecycleService.DATA_STREAM_MERGE_POLICY_TARGET_FACTOR_SETTING); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index 27fe65ba309d3..23a86b657b82d 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -38,6 +38,7 @@ public class DataStreamIndexSettingsProviderTests extends ESTestCase { + private static final TimeValue DEFAULT_LOOK_BACK_TIME = TimeValue.timeValueHours(2); // default private static final TimeValue DEFAULT_LOOK_AHEAD_TIME = TimeValue.timeValueHours(2); // default DataStreamIndexSettingsProvider provider; @@ -83,7 +84,7 @@ public void testGetAdditionalIndexSettings() throws Exception { List.of(new CompressedXContent(mapping)) ); assertThat(result.size(), equalTo(3)); - assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("field3")); } @@ -235,10 +236,31 @@ public void testGetAdditionalIndexSettingsLookAheadTime() throws Exception { List.of(new CompressedXContent("{}")) ); assertThat(result.size(), equalTo(2)); - assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis()))); } + public void testGetAdditionalIndexSettingsLookBackTime() throws Exception { + Metadata metadata = Metadata.EMPTY_METADATA; + String dataStreamName = "logs-app1"; + + Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); + TimeValue lookBackTime = TimeValue.timeValueHours(12); + Settings settings = builder().put("index.mode", "time_series").put("index.look_back_time", lookBackTime.getStringRep()).build(); + Settings result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + dataStreamName, + true, + metadata, + now, + settings, + List.of(new CompressedXContent("{}")) + ); + assertThat(result.size(), equalTo(2)); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookBackTime.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); + } + public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() throws Exception { String dataStreamName = "logs-app1"; TimeValue lookAheadTime = TimeValue.timeValueHours(2); @@ -358,7 +380,7 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() { List.of() ); assertThat(result.size(), equalTo(2)); - assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); + assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_BACK_TIME.getMillis()))); assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis()))); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/LookAHeadTimeTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/LookAHeadTimeTests.java index a0ed1a83d0de1..a612587262463 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/LookAHeadTimeTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/LookAHeadTimeTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.After; -import java.io.IOException; import java.util.Collection; import java.util.List; @@ -52,7 +51,7 @@ public void testTimeSeriesPollIntervalSettingToHigh() { assertThat(e.getMessage(), equalTo("failed to parse value [11m] for setting [time_series.poll_interval], must be <= [10m]")); } - public void testLookAheadTimeSetting() throws IOException { + public void testLookAheadTimeSetting() { var settings = Settings.builder().put(DataStreamsPlugin.LOOK_AHEAD_TIME.getKey(), "10m").build(); updateIndexSettings(settings); } @@ -69,6 +68,18 @@ public void testLookAheadTimeSettingToHigh() { assertThat(e.getMessage(), equalTo("failed to parse value [8d] for setting [index.look_ahead_time], must be <= [7d]")); } + public void testLookBackTimeSettingToLow() { + var settings = Settings.builder().put(DataStreamsPlugin.LOOK_BACK_TIME.getKey(), "1s").build(); + var e = expectThrows(IllegalArgumentException.class, () -> updateIndexSettings(settings)); + assertThat(e.getMessage(), equalTo("failed to parse value [1s] for setting [index.look_back_time], must be >= [1m]")); + } + + public void testLookBackTimeSettingToHigh() { + var settings = Settings.builder().put(DataStreamsPlugin.LOOK_BACK_TIME.getKey(), "8d").build(); + var e = expectThrows(IllegalArgumentException.class, () -> updateIndexSettings(settings)); + assertThat(e.getMessage(), equalTo("failed to parse value [8d] for setting [index.look_back_time], must be <= [7d]")); + } + public void testLookAheadTimeSettingLowerThanTimeSeriesPollIntervalSetting() { { var settings = Settings.builder() @@ -99,7 +110,7 @@ public void testLookAheadTimeSettingLowerThanTimeSeriesPollIntervalSetting() { } } - public void testLookAheadTimeSettingHigherThanTimeSeriesPollIntervalSetting() throws IOException { + public void testLookAheadTimeSettingHigherThanTimeSeriesPollIntervalSetting() { var clusterSettings = Settings.builder().put(DataStreamsPlugin.TIME_SERIES_POLL_INTERVAL.getKey(), "10m").build(); updateClusterSettings(clusterSettings); var indexSettings = Settings.builder().put(DataStreamsPlugin.LOOK_AHEAD_TIME.getKey(), "100m").build(); @@ -110,7 +121,7 @@ private void updateClusterSettings(Settings settings) { clusterAdmin().updateSettings(new ClusterUpdateSettingsRequest().persistentSettings(settings)).actionGet(); } - private void updateIndexSettings(Settings settings) throws IOException { + private void updateIndexSettings(Settings settings) { try { createIndex("test"); } catch (ResourceAlreadyExistsException e) { From 88e2508557ee2f76942a68188276f14825b39d9b Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 8 Sep 2023 10:36:10 +0100 Subject: [PATCH 24/69] Migrate ListenerTimeouts away from name-based executor (#99334) Replaces the executor name with a proper executor. Relates #99027 and friends --- .../action/support/ListenerTimeouts.java | 19 ++++++++++++++++++- .../action/support/ListenerTimeoutsTests.java | 12 ++++++++---- .../xpack/ccr/repository/CcrRepository.java | 10 +++++++--- .../TransportGetMlAutoscalingStats.java | 6 +++++- .../ql/async/AsyncTaskManagementService.java | 2 +- 5 files changed, 39 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java b/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java index 98c19fa5174dc..a4c66f6e0b541 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java +++ b/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java @@ -14,6 +14,7 @@ import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; @@ -35,7 +36,7 @@ public static ActionListener wrapWithTimeout( ThreadPool threadPool, ActionListener listener, TimeValue timeout, - String executor, + Executor executor, String listenerName ) { return wrapWithTimeout(threadPool, timeout, executor, listener, (ignore) -> { @@ -55,6 +56,22 @@ public static ActionListener wrapWithTimeout( * @param onTimeout consumer will be called and the resulting wrapper will be passed to it as a parameter * @return the wrapped listener that will timeout */ + public static ActionListener wrapWithTimeout( + ThreadPool threadPool, + TimeValue timeout, + Executor executor, + ActionListener listener, + Consumer> onTimeout + ) { + TimeoutableListener wrappedListener = new TimeoutableListener<>(listener, onTimeout); + wrappedListener.cancellable = threadPool.schedule(wrappedListener, timeout, executor); + return wrappedListener; + } + + /** + * @deprecated Use {@link #wrapWithTimeout(ThreadPool, TimeValue, Executor, ActionListener, Consumer)} instead. + */ + @Deprecated(forRemoval = true) public static ActionListener wrapWithTimeout( ThreadPool threadPool, TimeValue timeout, diff --git a/server/src/test/java/org/elasticsearch/action/support/ListenerTimeoutsTests.java b/server/src/test/java/org/elasticsearch/action/support/ListenerTimeoutsTests.java index ca959f47fd105..deaa7e5d31373 100644 --- a/server/src/test/java/org/elasticsearch/action/support/ListenerTimeoutsTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/ListenerTimeoutsTests.java @@ -17,6 +17,7 @@ import org.junit.Before; import java.io.IOException; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -25,13 +26,16 @@ public class ListenerTimeoutsTests extends ESTestCase { private final TimeValue timeout = TimeValue.timeValueMillis(10); - private final String generic = ThreadPool.Names.GENERIC; private DeterministicTaskQueue taskQueue; + private ThreadPool threadPool; + private Executor timeoutExecutor; @Before public void setUp() throws Exception { super.setUp(); taskQueue = new DeterministicTaskQueue(); + threadPool = taskQueue.getThreadPool(); + timeoutExecutor = threadPool.generic(); } public void testListenerTimeout() { @@ -39,7 +43,7 @@ public void testListenerTimeout() { AtomicReference exception = new AtomicReference<>(); ActionListener listener = wrap(success, exception); - ActionListener wrapped = ListenerTimeouts.wrapWithTimeout(taskQueue.getThreadPool(), listener, timeout, generic, "test"); + ActionListener wrapped = ListenerTimeouts.wrapWithTimeout(threadPool, listener, timeout, timeoutExecutor, "test"); assertTrue(taskQueue.hasDeferredTasks()); taskQueue.advanceTime(); taskQueue.runAllRunnableTasks(); @@ -56,7 +60,7 @@ public void testFinishNormallyBeforeTimeout() { AtomicReference exception = new AtomicReference<>(); ActionListener listener = wrap(success, exception); - ActionListener wrapped = ListenerTimeouts.wrapWithTimeout(taskQueue.getThreadPool(), listener, timeout, generic, "test"); + ActionListener wrapped = ListenerTimeouts.wrapWithTimeout(threadPool, listener, timeout, timeoutExecutor, "test"); wrapped.onResponse(null); wrapped.onFailure(new IOException("boom")); wrapped.onResponse(null); @@ -74,7 +78,7 @@ public void testFinishExceptionallyBeforeTimeout() { AtomicReference exception = new AtomicReference<>(); ActionListener listener = wrap(success, exception); - ActionListener wrapped = ListenerTimeouts.wrapWithTimeout(taskQueue.getThreadPool(), listener, timeout, generic, "test"); + ActionListener wrapped = ListenerTimeouts.wrapWithTimeout(threadPool, listener, timeout, timeoutExecutor, "test"); wrapped.onFailure(new IOException("boom")); assertTrue(taskQueue.hasDeferredTasks()); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 9dfea4a6c9358..fa8f8099900ce 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -583,6 +583,7 @@ void openSession( response.getStoreFileMetadata(), response.getMappingVersion(), threadPool, + chunkResponseExecutor, ccrSettings, throttledTime::inc, leaderShardId @@ -595,7 +596,7 @@ void openSession( threadPool, responseListener, ccrSettings.getRecoveryActionTimeout(), - ThreadPool.Names.GENERIC, + threadPool.generic(), // TODO should be the remote-client response executor to match the non-timeout case PutCcrRestoreSessionAction.INTERNAL_NAME ) ); @@ -611,6 +612,7 @@ private static class RestoreSession extends FileRestoreContext { private final CcrSettings ccrSettings; private final LongConsumer throttleListener; private final ThreadPool threadPool; + private final Executor timeoutExecutor; private final ShardId leaderShardId; RestoreSession( @@ -623,6 +625,7 @@ private static class RestoreSession extends FileRestoreContext { Store.MetadataSnapshot sourceMetadata, long mappingVersion, ThreadPool threadPool, + Executor timeoutExecutor, CcrSettings ccrSettings, LongConsumer throttleListener, ShardId leaderShardId @@ -634,6 +637,7 @@ private static class RestoreSession extends FileRestoreContext { this.sourceMetadata = sourceMetadata; this.mappingVersion = mappingVersion; this.threadPool = threadPool; + this.timeoutExecutor = timeoutExecutor; this.ccrSettings = ccrSettings; this.throttleListener = throttleListener; this.leaderShardId = leaderShardId; @@ -685,7 +689,7 @@ protected void executeChunkRequest(FileChunk request, ActionListener liste ListenerTimeouts.wrapWithTimeout(threadPool, listener.map(getCcrRestoreFileChunkResponse -> { writeFileChunk(request.md, getCcrRestoreFileChunkResponse); return null; - }), ccrSettings.getRecoveryActionTimeout(), ThreadPool.Names.GENERIC, GetCcrRestoreFileChunkAction.INTERNAL_NAME) + }), ccrSettings.getRecoveryActionTimeout(), timeoutExecutor, GetCcrRestoreFileChunkAction.INTERNAL_NAME) ); } @@ -740,7 +744,7 @@ public void close(ActionListener listener) { threadPool, listener, ccrSettings.getRecoveryActionTimeout(), - ThreadPool.Names.GENERIC, + timeoutExecutor, ClearCcrRestoreSessionAction.INTERNAL_NAME ); ClearCcrRestoreSessionRequest clearRequest = new ClearCcrRestoreSessionRequest(sessionUUID, node, leaderShardId); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java index c2acd97e53c95..7ce5fd0a66eb2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetMlAutoscalingStats.java @@ -33,6 +33,8 @@ import org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingResourceTracker; import org.elasticsearch.xpack.ml.process.MlMemoryTracker; +import java.util.concurrent.Executor; + /** * Internal (no-REST) transport to retrieve metrics for serverless autoscaling. */ @@ -41,6 +43,7 @@ public class TransportGetMlAutoscalingStats extends TransportMasterNodeAction MlAutoscalingResourceTracker.getMlAutoscalingStats( state, diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/AsyncTaskManagementService.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/AsyncTaskManagementService.java index e065b3814f292..29705d9e4b116 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/AsyncTaskManagementService.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/async/AsyncTaskManagementService.java @@ -324,7 +324,7 @@ public static listener.onResponse(new StoredAsyncResponse<>(r, task.getExpirationTimeMillis())), e -> listener.onResponse(new StoredAsyncResponse<>(e, task.getExpirationTimeMillis())) From 34711e73c099d3fce0332c945486a4e832fa55ba Mon Sep 17 00:00:00 2001 From: paras Date: Fri, 8 Sep 2023 15:11:10 +0530 Subject: [PATCH 25/69] [ML] Use more efficient buffering strategy for job input created by datafeeds (#98915) Java's own `ByteArrayOutputStream` doubles the size of its buffer when it needs to grow. This can be inefficient and lead to risk of OOMs with small JVM heaps. For example, suppose we need to build `autodetect` input 16.1MB in size. When the input grows past 16MB the buffer size will increase to 32MB, with a temporary requirement for 48MB at the point where both buffers exist. This change switches to use Elasticsearch's `BytesStreamOutput` class, which manages memory more efficiently. --- docs/changelog/98915.yaml | 5 +++++ .../extractor/scroll/ScrollDataExtractor.java | 19 +++---------------- 2 files changed, 8 insertions(+), 16 deletions(-) create mode 100644 docs/changelog/98915.yaml diff --git a/docs/changelog/98915.yaml b/docs/changelog/98915.yaml new file mode 100644 index 0000000000000..c23ddcc55d98e --- /dev/null +++ b/docs/changelog/98915.yaml @@ -0,0 +1,5 @@ +pr: 98915 +summary: Avoid risk of OOM in datafeeds when memory is constrained +area: Machine Learning +type: bug +issues: [89769] diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java index d70d34126fe27..e7aba2211b2df 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.search.SearchScrollAction; import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -30,8 +31,6 @@ import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter; import org.elasticsearch.xpack.ml.extractor.ExtractedField; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.NoSuchElementException; @@ -169,18 +168,6 @@ private SearchRequestBuilder buildSearchRequest(long start) { return searchRequestBuilder; } - /** - * Utility class to convert ByteArrayOutputStream to ByteArrayInputStream without copying the underlying buffer. - */ - private static class ConvertableByteArrayOutputStream extends ByteArrayOutputStream { - public ByteArrayInputStream resetThisAndGetByteArrayInputStream() { - ByteArrayInputStream inputStream = new ByteArrayInputStream(buf, 0, count); - buf = new byte[0]; - count = 0; - return inputStream; - } - } - /** * IMPORTANT: This is not an idempotent method. This method changes the input array by setting each element to null. */ @@ -192,7 +179,7 @@ private InputStream processAndConsumeSearchHits(SearchHit hits[]) throws IOExcep return null; } - ConvertableByteArrayOutputStream outputStream = new ConvertableByteArrayOutputStream(); + BytesStreamOutput outputStream = new BytesStreamOutput(); SearchHit lastHit = hits[hits.length - 1]; lastTimestamp = context.extractedFields.timeFieldValue(lastHit); @@ -217,7 +204,7 @@ private InputStream processAndConsumeSearchHits(SearchHit hits[]) throws IOExcep hits[i] = null; } } - return outputStream.resetThisAndGetByteArrayInputStream(); + return outputStream.bytes().streamInput(); } private InputStream continueScroll() throws IOException { From 93b56d7324fd6c1bfffb7e5dedcca73fe1859876 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 8 Sep 2023 12:57:28 +0200 Subject: [PATCH 26/69] Add dedicated pipeline for ecs dynamic template tests (#99353) Related to work done by https://github.com/elastic/elasticsearch/pull/97901 --- .../pipelines/ecs-dynamic-template-tests.yml | 9 +++++ catalog-info.yaml | 34 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 .buildkite/pipelines/ecs-dynamic-template-tests.yml diff --git a/.buildkite/pipelines/ecs-dynamic-template-tests.yml b/.buildkite/pipelines/ecs-dynamic-template-tests.yml new file mode 100644 index 0000000000000..b1fe972b724f1 --- /dev/null +++ b/.buildkite/pipelines/ecs-dynamic-template-tests.yml @@ -0,0 +1,9 @@ +steps: + - label: ecs-dynamic-templates-tests + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true :x-pack:plugin:stack:javaRestTest + timeout_in_minutes: 420 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + diskSizeGb: 350 + machineType: custom-32-98304 diff --git a/catalog-info.yaml b/catalog-info.yaml index dd2f3c105026e..b9c0c98b45e6c 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -269,3 +269,37 @@ spec: branch: lucene_snapshot cronline: "0 9,12,15,18 * * * America/New_York" message: "Runs tests against lucene_snapshot branch several times per day" +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-elasticsearch-ecs-dynamic-template-tests + description: Runs ECS dynamic template tests against main branch + links: + - title: Pipeline + url: https://buildkite.com/elastic/elasticsearch-ecs-dynamic-template-tests +spec: + type: buildkite-pipeline + system: buildkite + owner: group:elasticsearch-team + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + description: ":elasticsearch: ECS dynamic template tests against main branch" + name: elasticsearch / ecs-dynamic-templates / tests + spec: + repository: elastic/elasticsearch + pipeline_file: .buildkite/pipelines/ecs-dynamic-template-tests.yml + provider_settings: + trigger_mode: none + teams: + elasticsearch-team: {} + ml-core: {} + everyone: + access_level: READ_ONLY + schedules: + Daily: + branch: main + cronline: "0 12 * * * America/New_York" From 443c53c6369a66a26f306c6debdf8801ba2825d1 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 8 Sep 2023 07:17:18 -0400 Subject: [PATCH 27/69] ESQL: Add type to layout (#99327) We want it in a few places. --- .../compute/operator/EvalBenchmark.java | 5 +- .../xpack/esql/evaluator/EvalMapper.java | 2 +- .../AbstractPhysicalOperationProviders.java | 24 ++-- .../xpack/esql/planner/DefaultLayout.java | 62 +++++++++ .../planner/EsPhysicalOperationProviders.java | 10 +- .../xpack/esql/planner/ExchangeLayout.java | 47 ++++--- .../xpack/esql/planner/Layout.java | 119 ++++++++---------- .../esql/planner/LocalExecutionPlanner.java | 62 ++++----- .../function/AbstractFunctionTestCase.java | 2 +- .../xpack/esql/planner/EvalMapperTests.java | 8 +- .../TestPhysicalOperationProviders.java | 6 +- 11 files changed, 191 insertions(+), 156 deletions(-) create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/DefaultLayout.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java index 1f8cf632be8a1..e805958d04e78 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java @@ -44,6 +44,7 @@ import org.openjdk.jmh.annotations.Warmup; import java.time.Duration; +import java.util.Arrays; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -133,9 +134,7 @@ private static FieldAttribute intField() { private static Layout layout(FieldAttribute... fields) { Layout.Builder layout = new Layout.Builder(); - for (FieldAttribute field : fields) { - layout.appendChannel(field.id()); - } + layout.append(Arrays.asList(fields)); return layout.build(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java index 8556fc54dcb82..2f8f09b9ff02b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java @@ -161,7 +161,7 @@ public Block eval(Page page) { return page.getBlock(channel); } } - int channel = layout.getChannel(attr.id()); + int channel = layout.get(attr.id()).channel(); return () -> new Attribute(channel); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java index c81fbcf989cd3..0e984b3b85b0b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java @@ -58,9 +58,9 @@ public final PhysicalOperation groupingPhysicalOperation( // append channels to the layout if (mode == AggregateExec.Mode.FINAL) { - layout.appendChannels(aggregates); + layout.append(aggregates); } else { - layout.appendChannels(aggregateMapper.mapNonGrouping(aggregates)); + layout.append(aggregateMapper.mapNonGrouping(aggregates)); } // create the agg factories aggregatesToFactory( @@ -87,8 +87,8 @@ public final PhysicalOperation groupingPhysicalOperation( if (groupAttribute == null) { throw new EsqlIllegalArgumentException("Unexpected non-named expression[{}] as grouping in [{}]", group, aggregateExec); } - Set grpAttribIds = new HashSet<>(); - grpAttribIds.add(groupAttribute.id()); + Layout.ChannelSet groupAttributeLayout = new Layout.ChannelSet(new HashSet<>(), groupAttribute.dataType()); + groupAttributeLayout.nameIds().add(groupAttribute.id()); /* * Check for aliasing in aggregates which occurs in two cases (due to combining project + stats): @@ -99,10 +99,9 @@ public final PhysicalOperation groupingPhysicalOperation( if (agg instanceof Alias a) { if (a.child() instanceof Attribute attr) { if (groupAttribute.id().equals(attr.id())) { - grpAttribIds.add(a.id()); + groupAttributeLayout.nameIds().add(a.id()); // TODO: investigate whether a break could be used since it shouldn't be possible to have multiple - // attributes - // pointing to the same attribute + // attributes pointing to the same attribute } // partial mode only // check if there's any alias used in grouping - no need for the final reduction since the intermediate data @@ -117,18 +116,19 @@ else if (mode == AggregateExec.Mode.PARTIAL) { } } } - layout.appendChannel(grpAttribIds); - groupSpecs.add(new GroupSpec(source.layout.getChannel(groupAttribute.id()), groupAttribute)); + layout.append(groupAttributeLayout); + Layout.ChannelAndType groupInput = source.layout.get(groupAttribute.id()); + groupSpecs.add(new GroupSpec(groupInput == null ? null : groupInput.channel(), groupAttribute)); } if (mode == AggregateExec.Mode.FINAL) { for (var agg : aggregates) { if (agg instanceof Alias alias && alias.child() instanceof AggregateFunction) { - layout.appendChannel(alias.id()); + layout.append(alias); } } } else { - layout.appendChannels(aggregateMapper.mapGrouping(aggregates)); + layout.append(aggregateMapper.mapGrouping(aggregates)); } // create the agg factories @@ -252,7 +252,7 @@ private void aggregatesToFactory( params[i] = aggParams.get(i).fold(); } - List inputChannels = sourceAttr.stream().map(NamedExpression::id).map(layout::getChannel).toList(); + List inputChannels = sourceAttr.stream().map(attr -> layout.get(attr.id()).channel()).toList(); assert inputChannels != null && inputChannels.size() > 0 && inputChannels.stream().allMatch(i -> i >= 0); if (aggregateFunction instanceof ToAggregator agg) { consumer.accept(new AggFunctionSupplierContext(agg.supplier(bigArrays, inputChannels), aggMode)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/DefaultLayout.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/DefaultLayout.java new file mode 100644 index 0000000000000..e3a520149108a --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/DefaultLayout.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.planner; + +import org.elasticsearch.xpack.ql.expression.NameId; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +class DefaultLayout implements Layout { + private final Map layout; + private final int numberOfChannels; + + DefaultLayout(Map layout, int numberOfChannels) { + this.layout = layout; + this.numberOfChannels = numberOfChannels; + } + + @Override + public ChannelAndType get(NameId id) { + return layout.get(id); + } + + /** + * @return the total number of channels in the layout. + */ + @Override + public int numberOfChannels() { + return numberOfChannels; + } + + @Override + public Map> inverse() { + Map> inverse = new HashMap<>(); + for (Map.Entry entry : layout.entrySet()) { + NameId key = entry.getKey(); + Integer value = entry.getValue().channel(); + inverse.computeIfAbsent(value, k -> new HashSet<>()).add(key); + } + return inverse; + } + + /** + * @return creates a builder to append to this layout. + */ + @Override + public Layout.Builder builder() { + return new Builder(numberOfChannels, layout); + } + + @Override + public String toString() { + return "Layout{" + "layout=" + layout + ", numberOfChannels=" + numberOfChannels + '}'; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 337d48b5cad5d..bae1980a3e856 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -61,7 +61,7 @@ public final PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fi PhysicalOperation op = source; for (Attribute attr : fieldExtractExec.attributesToExtract()) { - layout.appendChannel(attr.id()); + layout.append(attr); Layout previousLayout = op.layout; var sources = ValueSources.sources( @@ -71,7 +71,7 @@ public final PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fi LocalExecutionPlanner.toElementType(attr.dataType()) ); - int docChannel = previousLayout.getChannel(sourceAttr.id()); + int docChannel = previousLayout.get(sourceAttr.id()).channel(); op = op.with( new ValuesSourceReaderOperator.ValuesSourceReaderOperatorFactory(sources, docChannel, attr.name()), @@ -137,9 +137,7 @@ public final PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec, ); } Layout.Builder layout = new Layout.Builder(); - for (int i = 0; i < esQueryExec.output().size(); i++) { - layout.appendChannel(esQueryExec.output().get(i).id()); - } + layout.append(esQueryExec.output()); int instanceCount = Math.max(1, luceneFactory.taskConcurrency()); context.driverParallelism(new DriverParallelism(DriverParallelism.Type.DATA_PARALLELISM, instanceCount)); return PhysicalOperation.fromSource(luceneFactory, layout.build()); @@ -155,7 +153,7 @@ public final Operator.OperatorFactory ordinalGroupingOperatorFactory( LocalExecutionPlannerContext context ) { var sourceAttribute = FieldExtractExec.extractSourceAttributesFrom(aggregateExec.child()); - int docChannel = source.layout.getChannel(sourceAttribute.id()); + int docChannel = source.layout.get(sourceAttribute.id()).channel(); // The grouping-by values are ready, let's group on them directly. // Costin: why are they ready and not already exposed in the layout? return new OrdinalsGroupingOperator.OrdinalsGroupingOperatorFactory( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/ExchangeLayout.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/ExchangeLayout.java index 327dc588e09b6..43bb36d070b79 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/ExchangeLayout.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/ExchangeLayout.java @@ -7,43 +7,32 @@ package org.elasticsearch.xpack.esql.planner; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.xpack.ql.expression.NameId; -import java.util.HashSet; +import java.util.HashMap; import java.util.Map; import java.util.Set; -import static java.util.Collections.emptyMap; - /** * Decorating layout that creates the NameId -> Value lazily based on the calls made to its content. * Essentially it maps the existing (old) NameIds to the new ones. */ -class ExchangeLayout extends Layout { - - private final Map delegate; +class ExchangeLayout implements Layout { + private final Layout delegate; private final Map> inverse; private final Map mappingToOldLayout; private int counter; - ExchangeLayout(Layout layout) { - super(emptyMap(), 0); - this.delegate = layout.internalLayout(); - this.mappingToOldLayout = Maps.newMapWithExpectedSize(delegate.size()); - this.inverse = Maps.newMapWithExpectedSize(delegate.size()); - - for (Map.Entry entry : delegate.entrySet()) { - NameId key = entry.getKey(); - Integer value = entry.getValue(); - inverse.computeIfAbsent(value, k -> new HashSet<>()).add(key); - } + ExchangeLayout(Layout delegate) { + this.delegate = delegate; + this.inverse = delegate.inverse(); + this.mappingToOldLayout = new HashMap<>(inverse.size()); } @Override - public Integer getChannel(NameId id) { + public ChannelAndType get(NameId id) { var oldId = mappingToOldLayout.get(id); - if (oldId == null && counter < delegate.size()) { + if (oldId == null && counter < inverse.size()) { var names = inverse.get(counter++); for (var name : names) { oldId = name; @@ -54,12 +43,22 @@ public Integer getChannel(NameId id) { } @Override - public int numberOfIds() { - return delegate.size(); + public int numberOfChannels() { + return delegate.numberOfChannels(); } @Override - public int numberOfChannels() { - return inverse.size(); + public String toString() { + return "ExchangeLayout{" + delegate + '}'; + } + + @Override + public Builder builder() { + throw new UnsupportedOperationException(); + } + + @Override + public Map> inverse() { + throw new UnsupportedOperationException(); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java index af7c94f45310f..a97a467aa3c0a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Layout.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.ql.expression.NameId; import org.elasticsearch.xpack.ql.expression.NamedExpression; +import org.elasticsearch.xpack.ql.type.DataType; import java.util.ArrayList; import java.util.Collection; @@ -18,119 +19,109 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.IntStream; /** * Maintains the mapping from attribute ids to channels (block index). * * An attribute can only be mapped to exactly one channel but one channel can be mapped to multiple attributes. */ -public class Layout { - - private final Map layout; - private final int numberOfChannels; - - Layout(Map layout, int numberOfChannels) { - this.layout = layout; - this.numberOfChannels = numberOfChannels; - } +public interface Layout { + /** + * The values stored in the {@link Layout}, a channel id and a {@link DataType}. + */ + record ChannelAndType(int channel, DataType type) {} /** - * @param id the attribute id - * @return the channel to which the specific attribute id is mapped or `null` if the attribute id does not exist in the layout. + * A part of an "inverse" layout, a {@link Set} or {@link NameId}s and a {@link DataType}. */ - public Integer getChannel(NameId id) { - return layout.get(id); - } + record ChannelSet(Set nameIds, DataType type) {} /** - * @return the total number of ids in the layout. + * @param id the attribute id + * @return the channel to which the specific attribute id is mapped or `null` if the attribute id does not exist in the layout. */ - public int numberOfIds() { - return layout.size(); - } + ChannelAndType get(NameId id); /** * @return the total number of channels in the layout. */ - public int numberOfChannels() { - return numberOfChannels; - } - - Map internalLayout() { - return layout; - } + int numberOfChannels(); /** * @return creates a builder to append to this layout. */ - public Layout.Builder builder() { - return new Layout.Builder(this); - } + Layout.Builder builder(); - @Override - public String toString() { - return "BlockLayout{" + "layout=" + layout + ", numberOfChannels=" + numberOfChannels + '}'; - } + Map> inverse(); /** * Builder class for Layout. The builder ensures that layouts cannot be altered after creation (through references to the underlying * map). */ - public static class Builder { + class Builder { + private final List channels = new ArrayList<>(); - private final List> channels; + public Builder() {} - public Builder() { - this.channels = new ArrayList<>(); - } - - private Builder(Layout layout) { - channels = IntStream.range(0, layout.numberOfChannels).>mapToObj(i -> new HashSet<>()).collect(Collectors.toList()); - for (Map.Entry entry : layout.layout.entrySet()) { - channels.get(entry.getValue()).add(entry.getKey()); + Builder(int numberOfChannels, Map layout) { + for (int i = 0; i < numberOfChannels; i++) { + channels.add(null); + } + for (Map.Entry entry : layout.entrySet()) { + ChannelSet set = channels.get(entry.getValue().channel); + if (set == null) { + set = new ChannelSet(new HashSet<>(), entry.getValue().type()); + channels.set(entry.getValue().channel, set); + } else { + if (set.type != entry.getValue().type()) { + throw new IllegalArgumentException(); + } + } + set.nameIds.add(entry.getKey()); } } /** - * Appends a new channel to the layout. The channel is mapped to a single attribute id. - * @param id the attribute id + * Appends a new channel to the layout. The channel is mapped to one or more attribute ids. */ - public Builder appendChannel(NameId id) { - channels.add(Set.of(id)); + public Builder append(ChannelSet set) { + if (set.nameIds.size() < 1) { + throw new IllegalArgumentException("Channel must be mapped to at least one id."); + } + channels.add(set); return this; } /** - * Appends a new channel to the layout. The channel is mapped to one or more attribute ids. - * @param ids the attribute ids + * Appends a new channel to the layout. The channel is mapped to a single attribute id. */ - public Builder appendChannel(Set ids) { - if (ids.size() < 1) { - throw new IllegalArgumentException("Channel must be mapped to at least one id."); - } - channels.add(ids); - return this; + public Builder append(NamedExpression attribute) { + return append(new ChannelSet(Set.of(attribute.id()), attribute.dataType())); } - public Builder appendChannels(Collection attributes) { - for (var attribute : attributes) { - appendChannel(attribute.id()); + /** + * Appends many new channels to the layout. Each channel is mapped to a single attribute id. + */ + public Builder append(Collection attributes) { + for (NamedExpression attribute : attributes) { + append(new ChannelSet(Set.of(attribute.id()), attribute.dataType())); } return this; } + /** + * Build a new {@link Layout}. + */ public Layout build() { - Map layout = new HashMap<>(); + Map layout = new HashMap<>(); int numberOfChannels = 0; - for (Set ids : this.channels) { + for (ChannelSet set : channels) { int channel = numberOfChannels++; - for (NameId id : ids) { - layout.putIfAbsent(id, channel); + for (NameId id : set.nameIds) { + layout.putIfAbsent(id, new ChannelAndType(channel, set.type)); } } - return new Layout(Collections.unmodifiableMap(layout), numberOfChannels); + return new DefaultLayout(Collections.unmodifiableMap(layout), numberOfChannels); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 74a6b561fb9b4..467e04deb579d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -281,7 +281,7 @@ private static Function alignPageToAttributes(List attrs, int index = -1; boolean transformRequired = false; for (var attribute : attrs) { - mappedPosition[++index] = layout.getChannel(attribute.id()); + mappedPosition[++index] = layout.get(attribute.id()).channel(); transformRequired |= mappedPosition[index] != index; } Function transformer = transformRequired ? p -> { @@ -314,9 +314,7 @@ private PhysicalOperation planExchangeSource(ExchangeSourceExec exchangeSource, Objects.requireNonNull(exchangeSourceHandler, "ExchangeSourceHandler wasn't provided"); var builder = new Layout.Builder(); - for (var attr : exchangeSource.output()) { - builder.appendChannel(attr.id()); - } + builder.append(exchangeSource.output()); // decorate the layout var l = builder.build(); var layout = exchangeSource.isIntermediateAgg() ? new ExchangeLayout(l) : l; @@ -330,7 +328,7 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte List orders = topNExec.order().stream().map(order -> { int sortByChannel; if (order.child() instanceof Attribute a) { - sortByChannel = source.layout.getChannel(a.id()); + sortByChannel = source.layout.get(a.id()).channel(); } else { throw new EsqlIllegalArgumentException("order by expression must be an attribute"); } @@ -386,7 +384,7 @@ private PhysicalOperation planEval(EvalExec eval, LocalExecutionPlannerContext c Supplier evaluatorSupplier; evaluatorSupplier = EvalMapper.toEvaluator(field.child(), source.layout); Layout.Builder layout = source.layout.builder(); - layout.appendChannel(field.toAttribute().id()); + layout.append(field.toAttribute()); source = source.with(new EvalOperatorFactory(evaluatorSupplier), layout.build()); } return source; @@ -395,9 +393,7 @@ private PhysicalOperation planEval(EvalExec eval, LocalExecutionPlannerContext c private PhysicalOperation planDissect(DissectExec dissect, LocalExecutionPlannerContext context) { PhysicalOperation source = plan(dissect.child(), context); Layout.Builder layoutBuilder = source.layout.builder(); - for (Attribute attr : dissect.extractedFields()) { - layoutBuilder.appendChannel(attr.id()); - } + layoutBuilder.append(dissect.extractedFields()); final Expression expr = dissect.inputExpression(); String[] attributeNames = Expressions.names(dissect.extractedFields()).toArray(new String[0]); @@ -417,10 +413,7 @@ private PhysicalOperation planGrok(GrokExec grok, LocalExecutionPlannerContext c PhysicalOperation source = plan(grok.child(), context); Layout.Builder layoutBuilder = source.layout.builder(); List extractedFields = grok.extractedFields(); - for (Attribute attr : extractedFields) { - layoutBuilder.appendChannel(attr.id()); - } - + layoutBuilder.append(extractedFields); Map fieldToPos = new HashMap<>(extractedFields.size()); Map fieldToType = new HashMap<>(extractedFields.size()); ElementType[] types = new ElementType[extractedFields.size()]; @@ -447,10 +440,7 @@ private PhysicalOperation planGrok(GrokExec grok, LocalExecutionPlannerContext c private PhysicalOperation planEnrich(EnrichExec enrich, LocalExecutionPlannerContext context) { PhysicalOperation source = plan(enrich.child(), context); Layout.Builder layoutBuilder = source.layout.builder(); - List extractedFields = enrich.enrichFields(); - for (NamedExpression attr : extractedFields) { - layoutBuilder.appendChannel(attr.id()); - } + layoutBuilder.append(enrich.enrichFields()); Layout layout = layoutBuilder.build(); Set indices = enrich.enrichIndex().concreteIndices(); if (indices.size() != 1) { @@ -462,7 +452,7 @@ private PhysicalOperation planEnrich(EnrichExec enrich, LocalExecutionPlannerCon sessionId, parentTask, 1, // TODO: Add a concurrent setting for enrich - also support unordered mode - source.layout.getChannel(enrich.matchField().id()), + source.layout.get(enrich.matchField().id()).channel(), enrichLookupService, enrichIndex, "match", // TODO: enrich should also resolve the match_type @@ -480,20 +470,13 @@ private Supplier toEvaluator(Expression exp, Layout layout) private PhysicalOperation planRow(RowExec row, LocalExecutionPlannerContext context) { List obj = row.fields().stream().map(f -> f.child().fold()).toList(); Layout.Builder layout = new Layout.Builder(); - var output = row.output(); - for (Attribute attribute : output) { - layout.appendChannel(attribute.id()); - } + layout.append(row.output()); return PhysicalOperation.fromSource(new RowOperatorFactory(obj), layout.build()); } private PhysicalOperation planLocal(LocalSourceExec localSourceExec, LocalExecutionPlannerContext context) { - Layout.Builder layout = new Layout.Builder(); - var output = localSourceExec.output(); - for (Attribute attribute : output) { - layout.appendChannel(attribute.id()); - } + layout.append(localSourceExec.output()); LocalSourceOperator.BlockSupplier supplier = () -> localSourceExec.supplier().get(); var operator = new LocalSourceOperator(supplier); return PhysicalOperation.fromSource(new LocalSourceFactory(() -> operator), layout.build()); @@ -501,16 +484,14 @@ private PhysicalOperation planLocal(LocalSourceExec localSourceExec, LocalExecut private PhysicalOperation planShow(ShowExec showExec) { Layout.Builder layout = new Layout.Builder(); - for (var attribute : showExec.output()) { - layout.appendChannel(attribute.id()); - } + layout.append(showExec.output()); return PhysicalOperation.fromSource(new ShowOperator.ShowOperatorFactory(showExec.values()), layout.build()); } private PhysicalOperation planProject(ProjectExec project, LocalExecutionPlannerContext context) { var source = plan(project.child(), context); - Map> inputChannelToOutputIds = new HashMap<>(); + Map inputChannelToOutputIds = new HashMap<>(); for (NamedExpression ne : project.projections()) { NameId inputId; if (ne instanceof Alias a) { @@ -518,19 +499,26 @@ private PhysicalOperation planProject(ProjectExec project, LocalExecutionPlanner } else { inputId = ne.id(); } - int inputChannel = source.layout.getChannel(inputId); - inputChannelToOutputIds.computeIfAbsent(inputChannel, ignore -> new HashSet<>()).add(ne.id()); + Layout.ChannelAndType input = source.layout.get(inputId); + Layout.ChannelSet channelSet = inputChannelToOutputIds.computeIfAbsent( + input.channel(), + ignore -> new Layout.ChannelSet(new HashSet<>(), input.type()) + ); + if (channelSet.type() != input.type()) { + throw new IllegalArgumentException("type mismatch for aliases"); + } + channelSet.nameIds().add(ne.id()); } BitSet mask = new BitSet(); Layout.Builder layout = new Layout.Builder(); for (int inChannel = 0; inChannel < source.layout.numberOfChannels(); inChannel++) { - Set outputIds = inputChannelToOutputIds.get(inChannel); + Layout.ChannelSet outputSet = inputChannelToOutputIds.get(inChannel); - if (outputIds != null) { + if (outputSet != null) { mask.set(inChannel); - layout.appendChannel(outputIds); + layout.append(outputSet); } } @@ -555,7 +543,7 @@ private PhysicalOperation planLimit(LimitExec limit, LocalExecutionPlannerContex private PhysicalOperation planMvExpand(MvExpandExec mvExpandExec, LocalExecutionPlannerContext context) { PhysicalOperation source = plan(mvExpandExec.child(), context); - return source.with(new MvExpandOperator.Factory(source.layout.getChannel(mvExpandExec.target().id())), source.layout); + return source.with(new MvExpandOperator.Factory(source.layout.get(mvExpandExec.target().id()).channel()), source.layout); } /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index ee752933311bf..a1a5e1aa60d19 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -149,7 +149,7 @@ protected final Page row(List values) { */ protected void buildLayout(Layout.Builder builder, Expression e) { if (e instanceof FieldAttribute f) { - builder.appendChannel(f.id()); + builder.append(f); return; } for (Expression c : e.children()) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java index ff41109bd3833..f42c86d4b028a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java @@ -121,10 +121,10 @@ public EvalMapperTests(String nodeString, Expression expression) { public void testEvaluatorSuppliers() { Layout.Builder lb = new Layout.Builder(); - lb.appendChannel(DOUBLE1.id()); - lb.appendChannel(DOUBLE2.id()); - lb.appendChannel(DATE.id()); - lb.appendChannel(LONG.id()); + lb.append(DOUBLE1); + lb.append(DOUBLE2); + lb.append(DATE); + lb.append(LONG); Layout layout = lb.build(); Supplier supplier = EvalMapper.toEvaluator(expression, layout); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index c088cae6f20c9..df163250e6e7a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -52,7 +52,7 @@ public PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fieldExt Layout.Builder layout = source.layout.builder(); PhysicalOperation op = source; for (Attribute attr : fieldExtractExec.attributesToExtract()) { - layout.appendChannel(attr.id()); + layout.append(attr); op = op.with(new TestFieldExtractOperatorFactory(attr.name()), layout.build()); } return op; @@ -61,9 +61,7 @@ public PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fieldExt @Override public PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec, LocalExecutionPlannerContext context) { Layout.Builder layout = new Layout.Builder(); - for (int i = 0; i < esQueryExec.output().size(); i++) { - layout.appendChannel(esQueryExec.output().get(i).id()); - } + layout.append(esQueryExec.output()); return PhysicalOperation.fromSource(new TestSourceOperatorFactory(), layout.build()); } From f5871af92949db9ce6e5a028e0b0cefe94cfeebe Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Fri, 8 Sep 2023 13:26:24 +0200 Subject: [PATCH 28/69] Enable debug logging in FileSettingsRoleMappingsStartupIT (#99247) investigating failing tests relates #98391 --- .../xpack/security/FileSettingsRoleMappingsStartupIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java index cea831ea6d0a9..07d11c6957407 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.SecurityIntegTestCase; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.transport.netty4.Netty4Plugin; import org.elasticsearch.xpack.wildcard.Wildcard; @@ -111,16 +112,19 @@ public void clusterChanged(ClusterChangedEvent event) { return new Tuple<>(savedClusterState, metadataVersion); } + @TestLogging(value = "org.elasticsearch.common.file:DEBUG", reason = "https://github.com/elastic/elasticsearch/issues/98391") public void testFailsOnStartMasterNodeWithError() throws Exception { internalCluster().setBootstrapMasterNodeIndex(0); internalCluster().startMasterOnlyNode(); + logger.info("--> write some role mappings, no other file settings"); writeJSONFile(internalCluster().getMasterName(), testJSONForFailedCase); var savedClusterState = setupClusterStateListenerForError(internalCluster().getMasterName()); boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); assertTrue(awaitSuccessful); + fail(); } public Collection> nodePlugins() { From af76a3a43611429b2c787bd446a0f9348f48ad2c Mon Sep 17 00:00:00 2001 From: Abdon Pijpelink Date: Fri, 8 Sep 2023 13:42:50 +0200 Subject: [PATCH 29/69] [DOCS] Add 'Troubleshooting an unstable cluster' to nav (#99287) * [DOCS] Add 'Troubleshooting an unstable cluster' to nav * Adjust docs links in code * Revert "Adjust docs links in code" This reverts commit f3846b1d7849a681a815f26245c8aa370bb6417b. --------- Co-authored-by: David Turner --- docs/reference/modules/discovery/fault-detection.asciidoc | 7 ++++++- docs/reference/troubleshooting.asciidoc | 6 ++++-- .../troubleshooting-unstable-cluster.asciidoc | 4 ++++ 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc diff --git a/docs/reference/modules/discovery/fault-detection.asciidoc b/docs/reference/modules/discovery/fault-detection.asciidoc index 001763430cf4b..dfa49e5b0d9af 100644 --- a/docs/reference/modules/discovery/fault-detection.asciidoc +++ b/docs/reference/modules/discovery/fault-detection.asciidoc @@ -35,7 +35,7 @@ starting from the beginning of the cluster state update. Refer to [[cluster-fault-detection-troubleshooting]] ==== Troubleshooting an unstable cluster - +//tag::troubleshooting[] Normally, a node will only leave a cluster if deliberately shut down. If a node leaves the cluster unexpectedly, it's important to address the cause. A cluster in which nodes leave unexpectedly is unstable and can create several issues. @@ -143,6 +143,7 @@ removes the node removed after three consecutively failed health checks. Refer to <> for information about the settings which control this mechanism. +[discrete] ===== Diagnosing `disconnected` nodes Nodes typically leave the cluster with reason `disconnected` when they shut @@ -181,6 +182,7 @@ In extreme cases, you may need to take packet captures using `tcpdump` to determine whether messages between nodes are being dropped or rejected by some other device on the network. +[discrete] ===== Diagnosing `lagging` nodes {es} needs every node to process cluster state updates reasonably quickly. If a @@ -225,6 +227,7 @@ To reconstruct the output, base64-decode the data and decompress it using cat lagdetector.log | sed -e 's/.*://' | base64 --decode | gzip --decompress ---- +[discrete] ===== Diagnosing `follower check retry count exceeded` nodes Nodes sometimes leave the cluster with reason `follower check retry count @@ -260,6 +263,7 @@ By default the follower checks will time out after 30s, so if node departures are unpredictable then capture stack dumps every 15s to be sure that at least one stack dump was taken at the right time. +[discrete] ===== Diagnosing `ShardLockObtainFailedException` failures If a node leaves and rejoins the cluster then {es} will usually shut down and @@ -295,3 +299,4 @@ To reconstruct the output, base64-decode the data and decompress it using ---- cat shardlock.log | sed -e 's/.*://' | base64 --decode | gzip --decompress ---- +//end::troubleshooting[] \ No newline at end of file diff --git a/docs/reference/troubleshooting.asciidoc b/docs/reference/troubleshooting.asciidoc index f263456d039d5..edd73ba393c38 100644 --- a/docs/reference/troubleshooting.asciidoc +++ b/docs/reference/troubleshooting.asciidoc @@ -48,8 +48,8 @@ fix problems that an {es} deployment might encounter. [discrete] [[troubleshooting-others]] -=== Others -* <> +=== Other issues +* <> * <> * <> * <> @@ -117,6 +117,8 @@ include::troubleshooting/snapshot/add-repository.asciidoc[] include::troubleshooting/snapshot/repeated-snapshot-failures.asciidoc[] +include::troubleshooting/troubleshooting-unstable-cluster.asciidoc[] + include::troubleshooting/discovery-issues.asciidoc[] include::monitoring/troubleshooting.asciidoc[] diff --git a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc new file mode 100644 index 0000000000000..387ebcdcd43c0 --- /dev/null +++ b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc @@ -0,0 +1,4 @@ +[[troubleshooting-unstable-cluster]] +== Troubleshooting an unstable cluster + +include::../modules/discovery/fault-detection.asciidoc[tag=troubleshooting,leveloffset=-2] \ No newline at end of file From f310cf396837185f9d9903a2b8d247d975e7bdc3 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 8 Sep 2023 12:43:15 +0100 Subject: [PATCH 30/69] Adjust links to troubleshooting docs in logs (#99354) In #99287 we introduced a new location for the unstable-cluster troubleshooting docs. This commit updates the links emitted in logs to point to the new location. --- .../org/elasticsearch/common/reference-docs-links.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json index c6dc459efa90b..ed1041acf3f54 100644 --- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json @@ -1,7 +1,7 @@ { "INITIAL_MASTER_NODES": "important-settings.html#initial_master_nodes", "DISCOVERY_TROUBLESHOOTING": "discovery-troubleshooting.html", - "UNSTABLE_CLUSTER_TROUBLESHOOTING": "cluster-fault-detection.html#cluster-fault-detection-troubleshooting", + "UNSTABLE_CLUSTER_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html", "LAGGING_NODE_TROUBLESHOOTING": "cluster-fault-detection.html#_diagnosing_lagging_nodes", "SHARD_LOCK_TROUBLESHOOTING": "cluster-fault-detection.html#_diagnosing_shardlockobtainfailedexception_failures", "CONCURRENT_REPOSITORY_WRITERS": "add-repository.html", From e4ecd25ef285643b2ad5ee85240d643837895057 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Fri, 8 Sep 2023 13:44:26 +0200 Subject: [PATCH 31/69] Remove accidental the Assert.fail (#99357) --- .../xpack/security/FileSettingsRoleMappingsStartupIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java index 07d11c6957407..7b02495c7227b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java @@ -124,7 +124,6 @@ public void testFailsOnStartMasterNodeWithError() throws Exception { boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); assertTrue(awaitSuccessful); - fail(); } public Collection> nodePlugins() { From aa3f0b6bd6483690ecdb9968af62e072f0cc2084 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 8 Sep 2023 14:30:16 +0200 Subject: [PATCH 32/69] [TEST] Address ExplainableScriptIT failure (#99351) ExplainableScriptIT tests the output of an explainable score script. The script declares that it does not need the score, in fact it returns the score as the value of a field from each document. The explain output though includes the explanation of the sub query, whose score is completely replaced by function score. The test assertson the sub query explanation which is thought inaccurate as the script declares that it needs no score. This test issue was made more evident by https://github.com/apache/lucene/pull/12383 as the sub query score became 0 . The solution is to not include the sub query explanation in the script explain output and remove the assertions that depend on that part of the output. --- .../search/functionscore/ExplainableScriptIT.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java index aa769fa565a87..08697bc1470fb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java @@ -16,7 +16,6 @@ import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.DocReader; @@ -43,7 +42,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutionException; import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; @@ -76,7 +74,7 @@ public boolean needs_score() { } @Override - public ScoreScript newInstance(DocReader docReader) throws IOException { + public ScoreScript newInstance(DocReader docReader) { return new MyScript(params1, lookup, ((DocValuesDocReader) docReader).getLeafReaderContext()); } }; @@ -99,13 +97,13 @@ static class MyScript extends ScoreScript implements ExplainableScoreScript { @Override public Explanation explain(Explanation subQueryScore) throws IOException { - Explanation scoreExp = Explanation.match(subQueryScore.getValue(), "_score: ", subQueryScore); - return Explanation.match((float) (execute(null)), "This script returned " + execute(null), scoreExp); + double score = execute(null); + return Explanation.match((float) score, "This script returned " + score); } @Override public double execute(ExplanationHolder explanation) { - return ((Number) ((ScriptDocValues) getDoc().get("number_field")).get(0)).doubleValue(); + return ((Number) (getDoc().get("number_field")).get(0)).doubleValue(); } } @@ -114,7 +112,7 @@ protected Collection> nodePlugins() { return Arrays.asList(ExplainableScriptPlugin.class); } - public void testExplainScript() throws InterruptedException, IOException, ExecutionException { + public void testExplainScript() throws InterruptedException, IOException { List indexRequests = new ArrayList<>(); for (int i = 0; i < 20; i++) { indexRequests.add( @@ -146,8 +144,6 @@ public void testExplainScript() throws InterruptedException, IOException, Execut for (SearchHit hit : hits.getHits()) { assertThat(hit.getId(), equalTo(Integer.toString(idCounter))); assertThat(hit.getExplanation().toString(), containsString(Double.toString(idCounter))); - assertThat(hit.getExplanation().toString(), containsString("1 = n")); - assertThat(hit.getExplanation().toString(), containsString("1 = N")); assertThat(hit.getExplanation().getDetails().length, equalTo(2)); idCounter--; } From d9b8b1393381dc10469b6a54b55e88fb5e30570b Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Fri, 8 Sep 2023 13:54:43 +0100 Subject: [PATCH 33/69] ESQL: Add memory Accountancy to Blocks and Vectors (#99235) This commit adds memory accountancy to Blocks and Vectors. This change is just the first step along the way to eventually add circuit breaking - we first need to the ability to determine ram usage, both before and after construction. Generally, we can track primitive array allocation and growth in builders separately (either before or after allocation), then use the block ramBytesUsed when releasing (giving back to the breaker). Care will be needed when calculating the actual ram bytes used before allocation (if needed), since that will need to take object reference and array header sizes into account, but seems reasonable. --- .../elasticsearch/common/util/BitArray.java | 12 +- .../compute/data/BooleanArrayBlock.java | 14 ++ .../compute/data/BooleanArrayVector.java | 13 + .../compute/data/BooleanBigArrayVector.java | 8 + .../compute/data/BooleanVectorBlock.java | 9 + .../compute/data/BytesRefArrayBlock.java | 13 + .../compute/data/BytesRefArrayVector.java | 12 + .../compute/data/BytesRefVectorBlock.java | 8 + .../compute/data/ConstantBooleanVector.java | 9 + .../compute/data/ConstantBytesRefVector.java | 8 + .../compute/data/ConstantDoubleVector.java | 9 + .../compute/data/ConstantIntVector.java | 9 + .../compute/data/ConstantLongVector.java | 9 + .../compute/data/DoubleArrayBlock.java | 14 ++ .../compute/data/DoubleArrayVector.java | 13 + .../compute/data/DoubleBigArrayVector.java | 8 + .../compute/data/DoubleVectorBlock.java | 9 + .../compute/data/FilterBooleanBlock.java | 11 + .../compute/data/FilterBooleanVector.java | 11 + .../compute/data/FilterBytesRefBlock.java | 10 + .../compute/data/FilterBytesRefVector.java | 10 + .../compute/data/FilterDoubleBlock.java | 11 + .../compute/data/FilterDoubleVector.java | 11 + .../compute/data/FilterIntBlock.java | 11 + .../compute/data/FilterIntVector.java | 11 + .../compute/data/FilterLongBlock.java | 11 + .../compute/data/FilterLongVector.java | 11 + .../compute/data/IntArrayBlock.java | 14 ++ .../compute/data/IntArrayVector.java | 13 + .../compute/data/IntBigArrayVector.java | 8 + .../compute/data/IntVectorBlock.java | 9 + .../compute/data/LongArrayBlock.java | 14 ++ .../compute/data/LongArrayVector.java | 13 + .../compute/data/LongBigArrayVector.java | 8 + .../compute/data/LongVectorBlock.java | 9 + .../compute/data/AbstractFilterVector.java | 2 +- .../org/elasticsearch/compute/data/Block.java | 3 +- .../compute/data/BlockRamUsageEstimator.java | 26 ++ .../compute/data/ConstantNullBlock.java | 8 + .../elasticsearch/compute/data/DocBlock.java | 9 + .../elasticsearch/compute/data/DocVector.java | 20 ++ .../elasticsearch/compute/data/Vector.java | 4 +- .../compute/data/X-ArrayBlock.java.st | 15 ++ .../compute/data/X-ArrayVector.java.st | 14 ++ .../compute/data/X-BigArrayVector.java.st | 8 + .../compute/data/X-ConstantVector.java.st | 8 + .../compute/data/X-FilterBlock.java.st | 10 + .../compute/data/X-FilterVector.java.st | 10 + .../compute/data/X-VectorBlock.java.st | 8 + .../compute/data/BlockAccountingTests.java | 230 ++++++++++++++++++ 50 files changed, 744 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java diff --git a/server/src/main/java/org/elasticsearch/common/util/BitArray.java b/server/src/main/java/org/elasticsearch/common/util/BitArray.java index 051160a81d1b0..696e81b3beec9 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BitArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/BitArray.java @@ -8,6 +8,8 @@ package org.elasticsearch.common.util; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -17,7 +19,10 @@ * The underlying long array grows lazily based on the biggest index * that needs to be set. */ -public final class BitArray implements Releasable { +public final class BitArray implements Accountable, Releasable { + + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BitArray.class); + private final BigArrays bigArrays; private LongArray bits; @@ -132,6 +137,11 @@ private static long bitmask(long index) { return 1L << index; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(bits); + } + @Override public void close() { Releasables.close(bits); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java index b0c2843554a69..dbd3580f90db8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java @@ -7,6 +7,8 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + import java.util.Arrays; import java.util.BitSet; import java.util.stream.IntStream; @@ -17,6 +19,8 @@ */ public final class BooleanArrayBlock extends AbstractArrayBlock implements BooleanBlock { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BooleanArrayBlock.class); + private final boolean[] values; public BooleanArrayBlock(boolean[] values, int positionCount, int[] firstValueIndexes, BitSet nulls, MvOrdering mvOrdering) { @@ -57,6 +61,16 @@ public BooleanBlock expand() { return new BooleanArrayBlock(values, end, firstValues, shiftNullsToExpandedPositions(), MvOrdering.UNORDERED); } + public static long ramBytesEstimated(boolean[] values, int[] firstValueIndexes, BitSet nullsMask) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values, firstValueIndexes, nullsMask); + } + @Override public boolean equals(Object obj) { if (obj instanceof BooleanBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java index 1b4374061b4e1..832b8f9f817bd 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java @@ -7,6 +7,8 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + import java.util.Arrays; /** @@ -15,6 +17,8 @@ */ public final class BooleanArrayVector extends AbstractVector implements BooleanVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BooleanArrayVector.class); + private final boolean[] values; public BooleanArrayVector(boolean[] values, int positionCount) { @@ -47,6 +51,15 @@ public BooleanVector filter(int... positions) { return new FilterBooleanVector(this, positions); } + public static long ramBytesEstimated(boolean[] values) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values); + } + @Override public boolean equals(Object obj) { if (obj instanceof BooleanVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java index d1f43310f00d1..25a34b383a4b4 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BitArray; import org.elasticsearch.core.Releasable; @@ -16,6 +17,8 @@ */ public final class BooleanBigArrayVector extends AbstractVector implements BooleanVector, Releasable { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BooleanBigArrayVector.class); + private final BitArray values; public BooleanBigArrayVector(BitArray values, int positionCount) { @@ -43,6 +46,11 @@ public boolean isConstant() { return false; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + @Override public BooleanVector filter(int... positions) { return new FilterBooleanVector(this, positions); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java index a89344019354b..4049aec5d9746 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Block view of a BooleanVector. * This class is generated. Do not edit it. */ public final class BooleanVectorBlock extends AbstractVectorBlock implements BooleanBlock { + private static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BooleanVectorBlock.class); + private final BooleanVector vector; BooleanVectorBlock(BooleanVector vector) { @@ -45,6 +49,11 @@ public BooleanBlock filter(int... positions) { return new FilterBooleanVector(vector, positions).asBlock(); } + @Override + public long ramBytesUsed() { + return RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector); + } + @Override public boolean equals(Object obj) { if (obj instanceof BooleanBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java index 263378e5cf846..38fba2f742bf3 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.data; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BytesRefArray; import java.util.BitSet; @@ -19,6 +20,8 @@ */ public final class BytesRefArrayBlock extends AbstractArrayBlock implements BytesRefBlock { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BytesRefArrayBlock.class); + private final BytesRefArray values; public BytesRefArrayBlock(BytesRefArray values, int positionCount, int[] firstValueIndexes, BitSet nulls, MvOrdering mvOrdering) { @@ -59,6 +62,16 @@ public BytesRefBlock expand() { return new BytesRefArrayBlock(values, end, firstValues, shiftNullsToExpandedPositions(), MvOrdering.UNORDERED); } + public static long ramBytesEstimated(BytesRefArray values, int[] firstValueIndexes, BitSet nullsMask) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values, firstValueIndexes, nullsMask); + } + @Override public boolean equals(Object obj) { if (obj instanceof BytesRefBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java index 6f1970fe66c38..42c92aa3be136 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayVector.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.data; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BytesRefArray; /** @@ -16,6 +17,8 @@ */ public final class BytesRefArrayVector extends AbstractVector implements BytesRefVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BytesRefArrayVector.class); + private final BytesRefArray values; public BytesRefArrayVector(BytesRefArray values, int positionCount) { @@ -48,6 +51,15 @@ public BytesRefVector filter(int... positions) { return new FilterBytesRefVector(this, positions); } + public static long ramBytesEstimated(BytesRefArray values) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values); + } + @Override public boolean equals(Object obj) { if (obj instanceof BytesRefVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java index 2136452af44b3..2b668ff34fe79 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.data; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; /** * Block view of a BytesRefVector. @@ -15,6 +16,8 @@ */ public final class BytesRefVectorBlock extends AbstractVectorBlock implements BytesRefBlock { + private static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BytesRefVectorBlock.class); + private final BytesRefVector vector; BytesRefVectorBlock(BytesRefVector vector) { @@ -47,6 +50,11 @@ public BytesRefBlock filter(int... positions) { return new FilterBytesRefVector(vector, positions).asBlock(); } + @Override + public long ramBytesUsed() { + return RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector); + } + @Override public boolean equals(Object obj) { if (obj instanceof BytesRefBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java index e802548350d39..3d6abc55d9469 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Vector implementation that stores a constant boolean value. * This class is generated. Do not edit it. */ public final class ConstantBooleanVector extends AbstractVector implements BooleanVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantBooleanVector.class); + private final boolean value; public ConstantBooleanVector(boolean value, int positionCount) { @@ -45,6 +49,11 @@ public boolean isConstant() { return true; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOfInstance(boolean.class); + } + @Override public boolean equals(Object obj) { if (obj instanceof BooleanVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBytesRefVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBytesRefVector.java index 25f07d72c1d65..896ac52bf0bc0 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBytesRefVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBytesRefVector.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.data; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; /** * Vector implementation that stores a constant BytesRef value. @@ -15,6 +16,8 @@ */ public final class ConstantBytesRefVector extends AbstractVector implements BytesRefVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantBytesRefVector.class); + private final BytesRef value; public ConstantBytesRefVector(BytesRef value, int positionCount) { @@ -47,6 +50,11 @@ public boolean isConstant() { return true; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class); + } + @Override public boolean equals(Object obj) { if (obj instanceof BytesRefVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantDoubleVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantDoubleVector.java index 8d196aa33f974..6099864b5b45b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantDoubleVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantDoubleVector.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Vector implementation that stores a constant double value. * This class is generated. Do not edit it. */ public final class ConstantDoubleVector extends AbstractVector implements DoubleVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantDoubleVector.class); + private final double value; public ConstantDoubleVector(double value, int positionCount) { @@ -45,6 +49,11 @@ public boolean isConstant() { return true; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOfInstance(double.class); + } + @Override public boolean equals(Object obj) { if (obj instanceof DoubleVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantIntVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantIntVector.java index ad942bb79e779..ab4e063c2ed78 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantIntVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantIntVector.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Vector implementation that stores a constant int value. * This class is generated. Do not edit it. */ public final class ConstantIntVector extends AbstractVector implements IntVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantIntVector.class); + private final int value; public ConstantIntVector(int value, int positionCount) { @@ -45,6 +49,11 @@ public boolean isConstant() { return true; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOfInstance(int.class); + } + @Override public boolean equals(Object obj) { if (obj instanceof IntVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantLongVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantLongVector.java index 79d9ba76db48c..c47c48182f1d1 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantLongVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantLongVector.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Vector implementation that stores a constant long value. * This class is generated. Do not edit it. */ public final class ConstantLongVector extends AbstractVector implements LongVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantLongVector.class); + private final long value; public ConstantLongVector(long value, int positionCount) { @@ -45,6 +49,11 @@ public boolean isConstant() { return true; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOfInstance(long.class); + } + @Override public boolean equals(Object obj) { if (obj instanceof LongVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java index c74de042da52b..0fd7aa987c315 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java @@ -7,6 +7,8 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + import java.util.Arrays; import java.util.BitSet; import java.util.stream.IntStream; @@ -17,6 +19,8 @@ */ public final class DoubleArrayBlock extends AbstractArrayBlock implements DoubleBlock { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DoubleArrayBlock.class); + private final double[] values; public DoubleArrayBlock(double[] values, int positionCount, int[] firstValueIndexes, BitSet nulls, MvOrdering mvOrdering) { @@ -57,6 +61,16 @@ public DoubleBlock expand() { return new DoubleArrayBlock(values, end, firstValues, shiftNullsToExpandedPositions(), MvOrdering.UNORDERED); } + public static long ramBytesEstimated(double[] values, int[] firstValueIndexes, BitSet nullsMask) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values, firstValueIndexes, nullsMask); + } + @Override public boolean equals(Object obj) { if (obj instanceof DoubleBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java index 340d434907643..2d2052371ed78 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayVector.java @@ -7,6 +7,8 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + import java.util.Arrays; /** @@ -15,6 +17,8 @@ */ public final class DoubleArrayVector extends AbstractVector implements DoubleVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DoubleArrayVector.class); + private final double[] values; public DoubleArrayVector(double[] values, int positionCount) { @@ -47,6 +51,15 @@ public DoubleVector filter(int... positions) { return new FilterDoubleVector(this, positions); } + public static long ramBytesEstimated(double[] values) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values); + } + @Override public boolean equals(Object obj) { if (obj instanceof DoubleVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayVector.java index 138fecbf0725b..129d4b3c31d93 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleBigArrayVector.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.DoubleArray; import org.elasticsearch.core.Releasable; @@ -16,6 +17,8 @@ */ public final class DoubleBigArrayVector extends AbstractVector implements DoubleVector, Releasable { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DoubleBigArrayVector.class); + private final DoubleArray values; public DoubleBigArrayVector(DoubleArray values, int positionCount) { @@ -43,6 +46,11 @@ public boolean isConstant() { return false; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + @Override public DoubleVector filter(int... positions) { return new FilterDoubleVector(this, positions); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java index d9fe827dff675..168cdc45167f6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Block view of a DoubleVector. * This class is generated. Do not edit it. */ public final class DoubleVectorBlock extends AbstractVectorBlock implements DoubleBlock { + private static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DoubleVectorBlock.class); + private final DoubleVector vector; DoubleVectorBlock(DoubleVector vector) { @@ -45,6 +49,11 @@ public DoubleBlock filter(int... positions) { return new FilterDoubleVector(vector, positions).asBlock(); } + @Override + public long ramBytesUsed() { + return RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector); + } + @Override public boolean equals(Object obj) { if (obj instanceof DoubleBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBooleanBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBooleanBlock.java index a3e3793498463..ed499489b3bb6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBooleanBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBooleanBlock.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Filter block for BooleanBlocks. * This class is generated. Do not edit it. */ final class FilterBooleanBlock extends AbstractFilterBlock implements BooleanBlock { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FilterBooleanBlock.class); + private final BooleanBlock block; FilterBooleanBlock(BooleanBlock block, int... positions) { @@ -65,6 +69,13 @@ public BooleanBlock expand() { return builder.build(); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter blocks encapsulate + // their inner block, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(block) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof BooleanBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBooleanVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBooleanVector.java index 5f6ad76e35a09..c519bc55dabd8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBooleanVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBooleanVector.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Filter vector for BooleanVectors. * This class is generated. Do not edit it. */ public final class FilterBooleanVector extends AbstractFilterVector implements BooleanVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FilterBooleanVector.class); + private final BooleanVector vector; FilterBooleanVector(BooleanVector vector, int... positions) { @@ -45,6 +49,13 @@ public BooleanVector filter(int... positions) { return new FilterBooleanVector(this, positions); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter vectors encapsulate + // their inner vector, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof BooleanVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBytesRefBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBytesRefBlock.java index 3bdd60dbedb2c..ad2266441fad7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBytesRefBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBytesRefBlock.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.data; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; /** * Filter block for BytesRefBlocks. @@ -15,6 +16,8 @@ */ final class FilterBytesRefBlock extends AbstractFilterBlock implements BytesRefBlock { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FilterBytesRefBlock.class); + private final BytesRefBlock block; FilterBytesRefBlock(BytesRefBlock block, int... positions) { @@ -69,6 +72,13 @@ public BytesRefBlock expand() { return builder.build(); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter blocks encapsulate + // their inner block, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(block) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof BytesRefBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBytesRefVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBytesRefVector.java index 63ef354fd6d36..3395621af9ccc 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBytesRefVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterBytesRefVector.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.data; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; /** * Filter vector for BytesRefVectors. @@ -15,6 +16,8 @@ */ public final class FilterBytesRefVector extends AbstractFilterVector implements BytesRefVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FilterBytesRefVector.class); + private final BytesRefVector vector; FilterBytesRefVector(BytesRefVector vector, int... positions) { @@ -47,6 +50,13 @@ public BytesRefVector filter(int... positions) { return new FilterBytesRefVector(this, positions); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter vectors encapsulate + // their inner vector, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof BytesRefVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterDoubleBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterDoubleBlock.java index 2f8f24b6b134f..51136e2c8def7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterDoubleBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterDoubleBlock.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Filter block for DoubleBlocks. * This class is generated. Do not edit it. */ final class FilterDoubleBlock extends AbstractFilterBlock implements DoubleBlock { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FilterDoubleBlock.class); + private final DoubleBlock block; FilterDoubleBlock(DoubleBlock block, int... positions) { @@ -65,6 +69,13 @@ public DoubleBlock expand() { return builder.build(); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter blocks encapsulate + // their inner block, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(block) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof DoubleBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterDoubleVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterDoubleVector.java index 6e841ec13b4e5..08ce7cefcd48a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterDoubleVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterDoubleVector.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Filter vector for DoubleVectors. * This class is generated. Do not edit it. */ public final class FilterDoubleVector extends AbstractFilterVector implements DoubleVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FilterDoubleVector.class); + private final DoubleVector vector; FilterDoubleVector(DoubleVector vector, int... positions) { @@ -45,6 +49,13 @@ public DoubleVector filter(int... positions) { return new FilterDoubleVector(this, positions); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter vectors encapsulate + // their inner vector, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof DoubleVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterIntBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterIntBlock.java index 21c3bb3ebdfbd..b915e40ab2d05 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterIntBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterIntBlock.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Filter block for IntBlocks. * This class is generated. Do not edit it. */ final class FilterIntBlock extends AbstractFilterBlock implements IntBlock { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FilterIntBlock.class); + private final IntBlock block; FilterIntBlock(IntBlock block, int... positions) { @@ -65,6 +69,13 @@ public IntBlock expand() { return builder.build(); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter blocks encapsulate + // their inner block, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(block) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof IntBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterIntVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterIntVector.java index 7caf0ee9ee45b..c4954318f0a99 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterIntVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterIntVector.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Filter vector for IntVectors. * This class is generated. Do not edit it. */ public final class FilterIntVector extends AbstractFilterVector implements IntVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FilterIntVector.class); + private final IntVector vector; FilterIntVector(IntVector vector, int... positions) { @@ -45,6 +49,13 @@ public IntVector filter(int... positions) { return new FilterIntVector(this, positions); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter vectors encapsulate + // their inner vector, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof IntVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterLongBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterLongBlock.java index d67d3e388b6ca..7461e5cbb0dc1 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterLongBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterLongBlock.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Filter block for LongBlocks. * This class is generated. Do not edit it. */ final class FilterLongBlock extends AbstractFilterBlock implements LongBlock { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FilterLongBlock.class); + private final LongBlock block; FilterLongBlock(LongBlock block, int... positions) { @@ -65,6 +69,13 @@ public LongBlock expand() { return builder.build(); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter blocks encapsulate + // their inner block, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(block) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof LongBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterLongVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterLongVector.java index 96d72f7959474..12dce9350e080 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterLongVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FilterLongVector.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Filter vector for LongVectors. * This class is generated. Do not edit it. */ public final class FilterLongVector extends AbstractFilterVector implements LongVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FilterLongVector.class); + private final LongVector vector; FilterLongVector(LongVector vector, int... positions) { @@ -45,6 +49,13 @@ public LongVector filter(int... positions) { return new FilterLongVector(this, positions); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter vectors encapsulate + // their inner vector, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof LongVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java index 2a52516148ab1..bc7f4fefdefb5 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java @@ -7,6 +7,8 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + import java.util.Arrays; import java.util.BitSet; import java.util.stream.IntStream; @@ -17,6 +19,8 @@ */ public final class IntArrayBlock extends AbstractArrayBlock implements IntBlock { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IntArrayBlock.class); + private final int[] values; public IntArrayBlock(int[] values, int positionCount, int[] firstValueIndexes, BitSet nulls, MvOrdering mvOrdering) { @@ -57,6 +61,16 @@ public IntBlock expand() { return new IntArrayBlock(values, end, firstValues, shiftNullsToExpandedPositions(), MvOrdering.UNORDERED); } + public static long ramBytesEstimated(int[] values, int[] firstValueIndexes, BitSet nullsMask) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values, firstValueIndexes, nullsMask); + } + @Override public boolean equals(Object obj) { if (obj instanceof IntBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java index c3a55e9e63075..a614f21c70b1d 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayVector.java @@ -7,6 +7,8 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + import java.util.Arrays; /** @@ -15,6 +17,8 @@ */ public final class IntArrayVector extends AbstractVector implements IntVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IntArrayVector.class); + private final int[] values; public IntArrayVector(int[] values, int positionCount) { @@ -47,6 +51,15 @@ public IntVector filter(int... positions) { return new FilterIntVector(this, positions); } + public static long ramBytesEstimated(int[] values) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values); + } + @Override public boolean equals(Object obj) { if (obj instanceof IntVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayVector.java index a172ea8b9cdc7..2058006eb45bb 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntBigArrayVector.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.IntArray; import org.elasticsearch.core.Releasable; @@ -16,6 +17,8 @@ */ public final class IntBigArrayVector extends AbstractVector implements IntVector, Releasable { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IntBigArrayVector.class); + private final IntArray values; public IntBigArrayVector(IntArray values, int positionCount) { @@ -43,6 +46,11 @@ public boolean isConstant() { return false; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + @Override public IntVector filter(int... positions) { return new FilterIntVector(this, positions); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java index 3be59882b6a3a..0d6d2e21bf36e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Block view of a IntVector. * This class is generated. Do not edit it. */ public final class IntVectorBlock extends AbstractVectorBlock implements IntBlock { + private static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(IntVectorBlock.class); + private final IntVector vector; IntVectorBlock(IntVector vector) { @@ -45,6 +49,11 @@ public IntBlock filter(int... positions) { return new FilterIntVector(vector, positions).asBlock(); } + @Override + public long ramBytesUsed() { + return RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector); + } + @Override public boolean equals(Object obj) { if (obj instanceof IntBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java index ec81eb4d59563..f973539e08009 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java @@ -7,6 +7,8 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + import java.util.Arrays; import java.util.BitSet; import java.util.stream.IntStream; @@ -17,6 +19,8 @@ */ public final class LongArrayBlock extends AbstractArrayBlock implements LongBlock { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(LongArrayBlock.class); + private final long[] values; public LongArrayBlock(long[] values, int positionCount, int[] firstValueIndexes, BitSet nulls, MvOrdering mvOrdering) { @@ -57,6 +61,16 @@ public LongBlock expand() { return new LongArrayBlock(values, end, firstValues, shiftNullsToExpandedPositions(), MvOrdering.UNORDERED); } + public static long ramBytesEstimated(long[] values, int[] firstValueIndexes, BitSet nullsMask) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values, firstValueIndexes, nullsMask); + } + @Override public boolean equals(Object obj) { if (obj instanceof LongBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java index 997412473af1b..dafe44b22415e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayVector.java @@ -7,6 +7,8 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + import java.util.Arrays; /** @@ -15,6 +17,8 @@ */ public final class LongArrayVector extends AbstractVector implements LongVector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(LongArrayVector.class); + private final long[] values; public LongArrayVector(long[] values, int positionCount) { @@ -47,6 +51,15 @@ public LongVector filter(int... positions) { return new FilterLongVector(this, positions); } + public static long ramBytesEstimated(long[] values) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values); + } + @Override public boolean equals(Object obj) { if (obj instanceof LongVector that) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayVector.java index 30c69a5792cb7..6db8675a8d69f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongBigArrayVector.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.LongArray; import org.elasticsearch.core.Releasable; @@ -16,6 +17,8 @@ */ public final class LongBigArrayVector extends AbstractVector implements LongVector, Releasable { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(LongBigArrayVector.class); + private final LongArray values; public LongBigArrayVector(LongArray values, int positionCount) { @@ -43,6 +46,11 @@ public boolean isConstant() { return false; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + @Override public LongVector filter(int... positions) { return new FilterLongVector(this, positions); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java index b9db747ec5ecf..330c10d6927fc 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java @@ -7,12 +7,16 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; + /** * Block view of a LongVector. * This class is generated. Do not edit it. */ public final class LongVectorBlock extends AbstractVectorBlock implements LongBlock { + private static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(LongVectorBlock.class); + private final LongVector vector; LongVectorBlock(LongVector vector) { @@ -45,6 +49,11 @@ public LongBlock filter(int... positions) { return new FilterLongVector(vector, positions).asBlock(); } + @Override + public long ramBytesUsed() { + return RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector); + } + @Override public boolean equals(Object obj) { if (obj instanceof LongBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractFilterVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractFilterVector.java index 03e73224564d9..a74ff44511602 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractFilterVector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractFilterVector.java @@ -15,7 +15,7 @@ */ abstract class AbstractFilterVector extends AbstractVector { - private final int[] positions; + protected final int[] positions; protected AbstractFilterVector(int[] positions) { super(positions.length); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java index 31c255196e9f7..f948b647d9c45 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.Accountable; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -30,7 +31,7 @@ * *

Block are immutable and can be passed between threads. */ -public interface Block extends NamedWriteable { +public interface Block extends Accountable, NamedWriteable { /** * {@return an efficient dense single-value view of this block}. diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java new file mode 100644 index 0000000000000..3d912a42f2a5e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockRamUsageEstimator.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.core.Nullable; + +import java.util.BitSet; + +public final class BlockRamUsageEstimator { + + /** Returns the size in bytes of the int[] object. Otherwise, returns 0 if null. */ + public static long sizeOf(@Nullable int[] arr) { + return arr == null ? 0 : RamUsageEstimator.sizeOf(arr); + } + + /** Returns the size in bytes used by the bitset. Otherwise, returns 0 if null. Not exact, but good enough */ + public static long sizeOfBitSet(@Nullable BitSet bitset) { + return bitset == null ? 0 : RamUsageEstimator.shallowSizeOfInstance(BitSet.class) + (bitset.size() / Byte.SIZE); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java index 5d5f65eb7ab1e..f3d26d443d2fa 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -19,6 +20,8 @@ */ public final class ConstantNullBlock extends AbstractBlock { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantNullBlock.class); + ConstantNullBlock(int positionCount) { super(positionCount); } @@ -93,6 +96,11 @@ public Block expand() { return this; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED; + } + @Override public boolean equals(Object obj) { if (obj instanceof ConstantNullBlock that) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java index 7d14241801352..364a8f413ef0f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; @@ -15,6 +16,9 @@ * Wrapper around {@link DocVector} to make a valid {@link Block}. */ public class DocBlock extends AbstractVectorBlock implements Block { + + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DocBlock.class); + private final DocVector vector; DocBlock(DocVector vector) { @@ -47,6 +51,11 @@ public Block filter(int... positions) { return new DocBlock(asVector().filter(positions)); } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector); + } + /** * A builder the for {@link DocBlock}. */ diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java index a17ab3d64a706..5227609ec71ee 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java @@ -8,11 +8,15 @@ package org.elasticsearch.compute.data; import org.apache.lucene.util.IntroSorter; +import org.apache.lucene.util.RamUsageEstimator; /** * {@link Vector} where each entry references a lucene document. */ public class DocVector extends AbstractVector implements Vector { + + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DocVector.class); + /** * Per position memory cost to build the shard segment doc map required * to load fields out of order. @@ -178,4 +182,20 @@ public ElementType elementType() { public boolean isConstant() { return shards.isConstant() && segments.isConstant() && docs.isConstant(); } + + public static long ramBytesEstimated( + IntVector shards, + IntVector segments, + IntVector docs, + int[] shardSegmentDocMapForwards, + int[] shardSegmentDocMapBackwards + ) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(shards) + RamUsageEstimator.sizeOf(segments) + RamUsageEstimator.sizeOf(docs) + + RamUsageEstimator.shallowSizeOf(shardSegmentDocMapForwards) + RamUsageEstimator.shallowSizeOf(shardSegmentDocMapBackwards); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(shards, segments, docs, shardSegmentDocMapForwards, shardSegmentDocMapBackwards); + } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java index 7954834a0debc..06997090ddbb4 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Vector.java @@ -7,10 +7,12 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.Accountable; + /** * A dense Vector of single values. */ -public interface Vector { +public interface Vector extends Accountable { /** * {@return Returns a Block view over this vector.} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st index 3f4b348185796..83a91a751e1b6 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st @@ -9,9 +9,12 @@ package org.elasticsearch.compute.data; $if(BytesRef)$ import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BytesRefArray; $else$ +import org.apache.lucene.util.RamUsageEstimator; + import java.util.Arrays; $endif$ import java.util.BitSet; @@ -23,6 +26,8 @@ import java.util.stream.IntStream; */ public final class $Type$ArrayBlock extends AbstractArrayBlock implements $Type$Block { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance($Type$ArrayBlock.class); + $if(BytesRef)$ private final BytesRefArray values; @@ -77,6 +82,16 @@ $endif$ return new $Type$ArrayBlock(values, end, firstValues, shiftNullsToExpandedPositions(), MvOrdering.UNORDERED); } + public static long ramBytesEstimated($if(BytesRef)$BytesRefArray$else$$type$[]$endif$ values, int[] firstValueIndexes, BitSet nullsMask) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values, firstValueIndexes, nullsMask); + } + @Override public boolean equals(Object obj) { if (obj instanceof $Type$Block that) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st index 4b3f234c05dc6..8b71ea69ab058 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st @@ -9,9 +9,12 @@ package org.elasticsearch.compute.data; $if(BytesRef)$ import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BytesRefArray; $else$ +import org.apache.lucene.util.RamUsageEstimator; + import java.util.Arrays; $endif$ @@ -21,6 +24,8 @@ $endif$ */ public final class $Type$ArrayVector extends AbstractVector implements $Type$Vector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance($Type$ArrayVector.class); + $if(BytesRef)$ private final BytesRefArray values; @@ -70,6 +75,15 @@ $endif$ return new Filter$Type$Vector(this, positions); } + public static long ramBytesEstimated($if(BytesRef)$BytesRefArray$else$$type$[]$endif$ values) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values); + } + @Override public boolean equals(Object obj) { if (obj instanceof $Type$Vector that) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st index 09566bed63dc3..153cf3c039145 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st @@ -7,6 +7,7 @@ package org.elasticsearch.compute.data; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.$if(boolean)$Bit$else$$Type$$endif$Array; import org.elasticsearch.core.Releasable; @@ -16,6 +17,8 @@ import org.elasticsearch.core.Releasable; */ public final class $Type$BigArrayVector extends AbstractVector implements $Type$Vector, Releasable { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance($Type$BigArrayVector.class); + private final $if(boolean)$Bit$else$$Type$$endif$Array values; public $Type$BigArrayVector($if(boolean)$Bit$else$$Type$$endif$Array values, int positionCount) { @@ -43,6 +46,11 @@ public final class $Type$BigArrayVector extends AbstractVector implements $Type$ return false; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + @Override public $Type$Vector filter(int... positions) { return new Filter$Type$Vector(this, positions); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st index 3915c0c0f7fbc..75a77da220435 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st @@ -10,6 +10,7 @@ package org.elasticsearch.compute.data; $if(BytesRef)$ import org.apache.lucene.util.BytesRef; $endif$ +import org.apache.lucene.util.RamUsageEstimator; /** * Vector implementation that stores a constant $type$ value. @@ -17,6 +18,8 @@ $endif$ */ public final class Constant$Type$Vector extends AbstractVector implements $Type$Vector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Constant$Type$Vector.class); + private final $type$ value; public Constant$Type$Vector($type$ value, int positionCount) { @@ -53,6 +56,11 @@ $endif$ return true; } + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.shallowSizeOfInstance($type$.class); + } + @Override public boolean equals(Object obj) { if (obj instanceof $Type$Vector that) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-FilterBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-FilterBlock.java.st index 097dfef0c6864..3dfaf02dc7c99 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-FilterBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-FilterBlock.java.st @@ -10,6 +10,7 @@ package org.elasticsearch.compute.data; $if(BytesRef)$ import org.apache.lucene.util.BytesRef; $endif$ +import org.apache.lucene.util.RamUsageEstimator; /** * Filter block for $Type$Blocks. @@ -17,6 +18,8 @@ $endif$ */ final class Filter$Type$Block extends AbstractFilterBlock implements $Type$Block { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Filter$Type$Block.class); + private final $Type$Block block; Filter$Type$Block($Type$Block block, int... positions) { @@ -82,6 +85,13 @@ $endif$ return builder.build(); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter blocks encapsulate + // their inner block, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(block) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof $Type$Block that) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-FilterVector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-FilterVector.java.st index 0f7c69805f406..bf7c7b399aa76 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-FilterVector.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-FilterVector.java.st @@ -10,6 +10,7 @@ package org.elasticsearch.compute.data; $if(BytesRef)$ import org.apache.lucene.util.BytesRef; $endif$ +import org.apache.lucene.util.RamUsageEstimator; /** * Filter vector for $Type$Vectors. @@ -17,6 +18,8 @@ $endif$ */ public final class Filter$Type$Vector extends AbstractFilterVector implements $Type$Vector { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Filter$Type$Vector.class); + private final $Type$Vector vector; Filter$Type$Vector($Type$Vector vector, int... positions) { @@ -54,6 +57,13 @@ $endif$ return new Filter$Type$Vector(this, positions); } + @Override + public long ramBytesUsed() { + // from a usage and resource point of view filter vectors encapsulate + // their inner vector, rather than listing it as a child resource + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector) + RamUsageEstimator.sizeOf(positions); + } + @Override public boolean equals(Object obj) { if (obj instanceof $Type$Vector that) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st index d9dd90f18a258..c6c8df46ba4e8 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st @@ -10,6 +10,7 @@ package org.elasticsearch.compute.data; $if(BytesRef)$ import org.apache.lucene.util.BytesRef; $endif$ +import org.apache.lucene.util.RamUsageEstimator; /** * Block view of a $Type$Vector. @@ -17,6 +18,8 @@ $endif$ */ public final class $Type$VectorBlock extends AbstractVectorBlock implements $Type$Block { + private static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance($Type$VectorBlock.class); + private final $Type$Vector vector; $Type$VectorBlock($Type$Vector vector) { @@ -54,6 +57,11 @@ $endif$ return new Filter$Type$Vector(vector, positions).asBlock(); } + @Override + public long ramBytesUsed() { + return RAM_BYTES_USED + RamUsageEstimator.sizeOf(vector); + } + @Override public boolean equals(Object obj) { if (obj instanceof $Type$Block that) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java new file mode 100644 index 0000000000000..5503c02be9794 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java @@ -0,0 +1,230 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.apache.lucene.tests.util.RamUsageTester; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArray; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BytesRefArray; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matcher; + +import java.lang.reflect.Field; +import java.util.BitSet; +import java.util.Collection; +import java.util.Map; + +import static org.apache.lucene.util.RamUsageEstimator.alignObjectSize; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class BlockAccountingTests extends ESTestCase { + + // A large(ish) upperbound simply so that effective greaterThan assertions are not unbounded + static final long UPPER_BOUND = 10_000; + + // Array Vectors + public void testBooleanVector() { + Vector empty = new BooleanArrayVector(new boolean[] {}, 0); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); + + Vector emptyPlusOne = new BooleanArrayVector(new boolean[] { randomBoolean() }, 1); + assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + 1))); + + boolean[] randomData = new boolean[randomIntBetween(1, 1024)]; + Vector emptyPlusSome = new BooleanArrayVector(randomData, randomData.length); + assertThat(emptyPlusSome.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + randomData.length))); + + // a filter becomes responsible for it's enclosing data, both in terms of accountancy and releasability + Vector filterVector = emptyPlusSome.filter(1); + assertThat(filterVector.ramBytesUsed(), between(emptyPlusSome.ramBytesUsed(), UPPER_BOUND)); + } + + public void testIntVector() { + Vector empty = new IntArrayVector(new int[] {}, 0); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); + + Vector emptyPlusOne = new IntArrayVector(new int[] { randomInt() }, 1); + assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Integer.BYTES))); + + int[] randomData = new int[randomIntBetween(1, 1024)]; + Vector emptyPlusSome = new IntArrayVector(randomData, randomData.length); + assertThat(emptyPlusSome.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + (long) Integer.BYTES * randomData.length))); + + // a filter becomes responsible for it's enclosing data, both in terms of accountancy and releasability + Vector filterVector = emptyPlusSome.filter(1); + assertThat(filterVector.ramBytesUsed(), between(emptyPlusSome.ramBytesUsed(), UPPER_BOUND)); + } + + public void testLongVector() { + Vector empty = new LongArrayVector(new long[] {}, 0); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); + + Vector emptyPlusOne = new LongArrayVector(new long[] { randomLong() }, 1); + assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + Long.BYTES)); + + long[] randomData = new long[randomIntBetween(1, 1024)]; + Vector emptyPlusSome = new LongArrayVector(randomData, randomData.length); + assertThat(emptyPlusSome.ramBytesUsed(), is(empty.ramBytesUsed() + (long) Long.BYTES * randomData.length)); + + // a filter becomes responsible for it's enclosing data, both in terms of accountancy and releasability + Vector filterVector = emptyPlusSome.filter(1); + assertThat(filterVector.ramBytesUsed(), between(emptyPlusSome.ramBytesUsed(), UPPER_BOUND)); + } + + public void testDoubleVector() { + Vector empty = new DoubleArrayVector(new double[] {}, 0); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); + + Vector emptyPlusOne = new DoubleArrayVector(new double[] { randomDouble() }, 1); + assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + Double.BYTES)); + + double[] randomData = new double[randomIntBetween(1, 1024)]; + Vector emptyPlusSome = new DoubleArrayVector(randomData, randomData.length); + assertThat(emptyPlusSome.ramBytesUsed(), is(empty.ramBytesUsed() + (long) Double.BYTES * randomData.length)); + + // a filter becomes responsible for it's enclosing data, both in terms of accountancy and releasability + Vector filterVector = emptyPlusSome.filter(1); + assertThat(filterVector.ramBytesUsed(), between(emptyPlusSome.ramBytesUsed(), UPPER_BOUND)); + } + + public void testBytesRefVector() { + try ( + var emptyArray = new BytesRefArray(0, BigArrays.NON_RECYCLING_INSTANCE); + var arrayWithOne = new BytesRefArray(0, BigArrays.NON_RECYCLING_INSTANCE) + ) { + var acc = new RamUsageTester.Accumulator() { + @Override + public long accumulateObject(Object o, long shallowSize, Map fieldValues, Collection queue) { + for (var entry : fieldValues.entrySet()) { + if (entry.getKey().getType().equals(BigArrays.class)) { + // skip BigArrays, as it is (correctly) not part of the ramBytesUsed for BytesRefArray + } else if (o instanceof BigArray bigArray) { + return bigArray.ramBytesUsed(); + } else { + queue.add(entry.getValue()); + } + } + return shallowSize; + } + }; + Vector emptyVector = new BytesRefArrayVector(emptyArray, 0); + long expectedEmptyVectorUsed = RamUsageTester.ramUsed(emptyVector, acc); + assertThat(emptyVector.ramBytesUsed(), is(expectedEmptyVectorUsed)); + + var bytesRef = new BytesRef(randomAlphaOfLengthBetween(1, 16)); + arrayWithOne.append(bytesRef); + Vector emptyPlusOne = new BytesRefArrayVector(arrayWithOne, 1); + assertThat(emptyPlusOne.ramBytesUsed(), between(emptyVector.ramBytesUsed() + bytesRef.length, UPPER_BOUND)); + + // a filter becomes responsible for it's enclosing data, both in terms of accountancy and releasability + Vector filterVector = emptyPlusOne.filter(1); + assertThat(filterVector.ramBytesUsed(), between(emptyPlusOne.ramBytesUsed(), UPPER_BOUND)); + } + } + + // Array Blocks + public void testBooleanBlock() { + Block empty = new BooleanArrayBlock(new boolean[] {}, 0, new int[] {}, null, Block.MvOrdering.UNORDERED); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); + + Block emptyPlusOne = new BooleanArrayBlock(new boolean[] { randomBoolean() }, 1, new int[] {}, null, Block.MvOrdering.UNORDERED); + assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + 1))); + + boolean[] randomData = new boolean[randomIntBetween(1, 1024)]; + Block emptyPlusSome = new BooleanArrayBlock(randomData, randomData.length, new int[] {}, null, Block.MvOrdering.UNORDERED); + assertThat(emptyPlusSome.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + randomData.length))); + + Block filterBlock = emptyPlusSome.filter(1); + assertThat(filterBlock.ramBytesUsed(), between(emptyPlusSome.ramBytesUsed(), UPPER_BOUND)); + } + + public void testBooleanBlockWithNullFirstValues() { + Block empty = new BooleanArrayBlock(new boolean[] {}, 0, null, BitSet.valueOf(new byte[] { 1 }), Block.MvOrdering.UNORDERED); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), lessThanOrEqualTo(expectedEmptyUsed)); + } + + public void testIntBlock() { + Block empty = new IntArrayBlock(new int[] {}, 0, new int[] {}, null, Block.MvOrdering.UNORDERED); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); + + Block emptyPlusOne = new IntArrayBlock(new int[] { randomInt() }, 1, new int[] {}, null, Block.MvOrdering.UNORDERED); + assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Integer.BYTES))); + + int[] randomData = new int[randomIntBetween(1, 1024)]; + Block emptyPlusSome = new IntArrayBlock(randomData, randomData.length, new int[] {}, null, Block.MvOrdering.UNORDERED); + assertThat(emptyPlusSome.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + (long) Integer.BYTES * randomData.length))); + + Block filterBlock = emptyPlusSome.filter(1); + assertThat(filterBlock.ramBytesUsed(), between(emptyPlusSome.ramBytesUsed(), UPPER_BOUND)); + } + + public void testIntBlockWithNullFirstValues() { + Block empty = new IntArrayBlock(new int[] {}, 0, null, BitSet.valueOf(new byte[] { 1 }), Block.MvOrdering.UNORDERED); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); + } + + public void testLongBlock() { + Block empty = new LongArrayBlock(new long[] {}, 0, new int[] {}, null, Block.MvOrdering.UNORDERED); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); + + Block emptyPlusOne = new LongArrayBlock(new long[] { randomInt() }, 1, new int[] {}, null, Block.MvOrdering.UNORDERED); + assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Long.BYTES))); + + long[] randomData = new long[randomIntBetween(1, 1024)]; + Block emptyPlusSome = new LongArrayBlock(randomData, randomData.length, new int[] {}, null, Block.MvOrdering.UNORDERED); + assertThat(emptyPlusSome.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + (long) Long.BYTES * randomData.length))); + + Block filterBlock = emptyPlusSome.filter(1); + assertThat(filterBlock.ramBytesUsed(), between(emptyPlusSome.ramBytesUsed(), UPPER_BOUND)); + } + + public void testLongBlockWithNullFirstValues() { + Block empty = new LongArrayBlock(new long[] {}, 0, null, BitSet.valueOf(new byte[] { 1 }), Block.MvOrdering.UNORDERED); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); + } + + public void testDoubleBlock() { + Block empty = new DoubleArrayBlock(new double[] {}, 0, new int[] {}, null, Block.MvOrdering.UNORDERED); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); + + Block emptyPlusOne = new DoubleArrayBlock(new double[] { randomInt() }, 1, new int[] {}, null, Block.MvOrdering.UNORDERED); + assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Double.BYTES))); + + double[] randomData = new double[randomIntBetween(1, 1024)]; + Block emptyPlusSome = new DoubleArrayBlock(randomData, randomData.length, new int[] {}, null, Block.MvOrdering.UNORDERED); + assertThat(emptyPlusSome.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + (long) Double.BYTES * randomData.length))); + + Block filterBlock = emptyPlusSome.filter(1); + assertThat(filterBlock.ramBytesUsed(), between(emptyPlusSome.ramBytesUsed(), UPPER_BOUND)); + } + + public void testDoubleBlockWithNullFirstValues() { + Block empty = new DoubleArrayBlock(new double[] {}, 0, null, BitSet.valueOf(new byte[] { 1 }), Block.MvOrdering.UNORDERED); + long expectedEmptyUsed = RamUsageTester.ramUsed(empty); + assertThat(empty.ramBytesUsed(), is(expectedEmptyUsed)); + } + + static Matcher between(long minInclusive, long maxInclusive) { + return allOf(greaterThanOrEqualTo(minInclusive), lessThanOrEqualTo(maxInclusive)); + } +} From f691eda50f80477094e146bd6a3bc358ed689003 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Fri, 8 Sep 2023 09:01:22 -0400 Subject: [PATCH 34/69] Pass compatibility versions from node to gateway meta state (#99321) In #99252, we passed CompatibilityVersions down from Node to join requests, so that when nodes join or leave the cluster, we accurately update compatibility versions in cluster state. However, we also need to pass a wrapped TransportVersion from Node down to GatewayMetaState so that we can initialize correct version values in cluster state. This will be important once we add system index mappings versions to CompatibilityVersions (#99307), as those cannot be determined statically. --- .../gateway/ClusterStateUpdaters.java | 10 +++-- .../gateway/GatewayMetaState.java | 44 ++++++++++++++----- .../java/org/elasticsearch/node/Node.java | 4 +- .../gateway/ClusterStateUpdatersTests.java | 6 +-- .../GatewayMetaStatePersistedStateTests.java | 4 +- .../gateway/GatewayMetaStateTests.java | 15 ++++++- .../gateway/MockGatewayMetaState.java | 15 +++++-- 7 files changed, 72 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java index 758690e41699a..2ffadfb26a985 100644 --- a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java +++ b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -19,6 +18,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy; +import org.elasticsearch.cluster.version.CompatibilityVersions; import org.elasticsearch.common.settings.ClusterSettings; import java.util.Map; @@ -29,10 +29,14 @@ public class ClusterStateUpdaters { private static final Logger logger = LogManager.getLogger(ClusterStateUpdaters.class); - public static ClusterState setLocalNode(ClusterState clusterState, DiscoveryNode localNode, TransportVersion transportVersion) { + public static ClusterState setLocalNode( + ClusterState clusterState, + DiscoveryNode localNode, + CompatibilityVersions compatibilityVersions + ) { return ClusterState.builder(clusterState) .nodes(DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()).build()) - .putTransportVersion(localNode.getId(), transportVersion) + .compatibilityVersions(Map.of(localNode.getId(), compatibilityVersions)) .build(); } diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 8fe86cc1a2ada..335b63d656b1a 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -14,7 +14,6 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -28,6 +27,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.version.CompatibilityVersions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -97,7 +97,8 @@ public void start( IndexMetadataVerifier indexMetadataVerifier, MetadataUpgrader metadataUpgrader, PersistedClusterStateService persistedClusterStateService, - List clusterCoordinationPlugins + List clusterCoordinationPlugins, + CompatibilityVersions compatibilityVersions ) { assert persistedState.get() == null : "should only start once, but already have " + persistedState.get(); try { @@ -110,7 +111,8 @@ public void start( indexMetadataVerifier, metadataUpgrader, persistedClusterStateService, - clusterCoordinationPlugins + clusterCoordinationPlugins, + compatibilityVersions ) ); } catch (IOException e) { @@ -126,7 +128,8 @@ private PersistedState createPersistedState( IndexMetadataVerifier indexMetadataVerifier, MetadataUpgrader metadataUpgrader, PersistedClusterStateService persistedClusterStateService, - List clusterCoordinationPlugins + List clusterCoordinationPlugins, + CompatibilityVersions compatibilityVersions ) throws IOException { final var persistedStateFactories = clusterCoordinationPlugins.stream() .map(ClusterCoordinationPlugin::getPersistedStateFactory) @@ -149,11 +152,19 @@ private PersistedState createPersistedState( metaStateService, indexMetadataVerifier, metadataUpgrader, - persistedClusterStateService + persistedClusterStateService, + compatibilityVersions ); } - return createInMemoryPersistedState(settings, transportService, clusterService, metaStateService, persistedClusterStateService); + return createInMemoryPersistedState( + settings, + transportService, + clusterService, + metaStateService, + persistedClusterStateService, + compatibilityVersions + ); } private PersistedState createOnDiskPersistedState( @@ -163,7 +174,8 @@ private PersistedState createOnDiskPersistedState( MetaStateService metaStateService, IndexMetadataVerifier indexMetadataVerifier, MetadataUpgrader metadataUpgrader, - PersistedClusterStateService persistedClusterStateService + PersistedClusterStateService persistedClusterStateService, + CompatibilityVersions compatibilityVersions ) throws IOException { final PersistedClusterStateService.OnDiskState onDiskState = persistedClusterStateService.loadBestOnDiskState(); @@ -190,7 +202,8 @@ private PersistedState createOnDiskPersistedState( ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings)) .version(lastAcceptedVersion) .metadata(upgradeMetadataForNode(metadata, indexMetadataVerifier, metadataUpgrader)) - .build() + .build(), + compatibilityVersions ); if (DiscoveryNode.isMasterNode(settings)) { persistedState = new LucenePersistedState(persistedClusterStateService, currentTerm, clusterState); @@ -226,13 +239,15 @@ private PersistedState createInMemoryPersistedState( TransportService transportService, ClusterService clusterService, MetaStateService metaStateService, - PersistedClusterStateService persistedClusterStateService + PersistedClusterStateService persistedClusterStateService, + CompatibilityVersions compatibilityVersions ) throws IOException { final long currentTerm = 0L; final ClusterState clusterState = prepareInitialClusterState( transportService, clusterService, - ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings)).build() + ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings)).build(), + compatibilityVersions ); if (persistedClusterStateService.getDataPaths().length > 0) { // write empty cluster state just so that we have a persistent node id. There is no need to write out global metadata with @@ -252,12 +267,17 @@ private PersistedState createInMemoryPersistedState( } // exposed so it can be overridden by tests - ClusterState prepareInitialClusterState(TransportService transportService, ClusterService clusterService, ClusterState clusterState) { + ClusterState prepareInitialClusterState( + TransportService transportService, + ClusterService clusterService, + ClusterState clusterState, + CompatibilityVersions compatibilityVersions + ) { assert clusterState.nodes().getLocalNode() == null : "prepareInitialClusterState must only be called once"; assert transportService.getLocalNode() != null : "transport service is not yet started"; return Function.identity() .andThen(ClusterStateUpdaters::addStateNotRecoveredBlock) - .andThen(state -> ClusterStateUpdaters.setLocalNode(state, transportService.getLocalNode(), TransportVersion.current())) + .andThen(state -> ClusterStateUpdaters.setLocalNode(state, transportService.getLocalNode(), compatibilityVersions)) .andThen(state -> ClusterStateUpdaters.upgradeAndArchiveUnknownOrInvalidSettings(state, clusterService.getClusterSettings())) .andThen(ClusterStateUpdaters::recoverClusterBlocks) .apply(clusterState); diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 37f18e36f0a6e..da0fb6064917c 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -1155,6 +1155,7 @@ protected Node( b.bind(FileSettingsService.class).toInstance(fileSettingsService); b.bind(WriteLoadForecaster.class).toInstance(writeLoadForecaster); b.bind(HealthPeriodicLogger.class).toInstance(healthPeriodicLogger); + b.bind(CompatibilityVersions.class).toInstance(compatibilityVersions); }); if (ReadinessService.enabled(environment)) { @@ -1478,7 +1479,8 @@ public Node start() throws NodeValidationException { injector.getInstance(IndexMetadataVerifier.class), injector.getInstance(MetadataUpgrader.class), injector.getInstance(PersistedClusterStateService.class), - pluginsService.filterPlugins(ClusterCoordinationPlugin.class) + pluginsService.filterPlugins(ClusterCoordinationPlugin.class), + injector.getInstance(CompatibilityVersions.class) ); // TODO: Do not expect that the legacy metadata file is always present https://github.com/elastic/elasticsearch/issues/95211 if (Assertions.ENABLED && DiscoveryNode.isStateless(settings()) == false) { diff --git a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java index 4ea843bb7ece0..5112556fe79af 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java @@ -7,7 +7,6 @@ */ package org.elasticsearch.gateway; -import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.TestShardRoutingRoleStrategies; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -18,6 +17,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.version.CompatibilityVersionsUtils; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; @@ -217,7 +217,7 @@ public void testSetLocalNode() { final ClusterState initialState = ClusterState.builder(ClusterState.EMPTY_STATE).metadata(metadata).build(); final DiscoveryNode localNode = DiscoveryNodeUtils.builder("node1").roles(Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE)).build(); - final ClusterState updatedState = setLocalNode(initialState, localNode, TransportVersion.current()); + final ClusterState updatedState = setLocalNode(initialState, localNode, CompatibilityVersionsUtils.staticCurrent()); assertMetadataEquals(initialState, updatedState); assertThat(updatedState.nodes().getLocalNode(), equalTo(localNode)); @@ -261,7 +261,7 @@ public void testHideStateIfNotRecovered() { .build(); final DiscoveryNode localNode = DiscoveryNodeUtils.builder("node1").roles(Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE)).build(); final ClusterState updatedState = Function.identity() - .andThen(state -> setLocalNode(state, localNode, TransportVersion.current())) + .andThen(state -> setLocalNode(state, localNode, CompatibilityVersionsUtils.staticCurrent())) .andThen(ClusterStateUpdaters::recoverClusterBlocks) .apply(initialState); diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java index d0c0ccc8a8804..547e4e4eccda4 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.version.CompatibilityVersionsUtils; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -401,7 +402,8 @@ public void testDataOnlyNodePersistence() throws Exception { null, null, persistedClusterStateService, - List.of() + List.of(), + CompatibilityVersionsUtils.staticCurrent() ); final CoordinationState.PersistedState persistedState = gateway.getPersistedState(); assertThat(persistedState, instanceOf(GatewayMetaState.AsyncPersistedState.class)); diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index b5aacd7f93fd5..22869ad37524c 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadataVerifier; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.version.CompatibilityVersionsUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.plugins.ClusterCoordinationPlugin; @@ -163,7 +164,17 @@ public Optional getPersistedStateFactory() { assertThat( expectThrows( IllegalStateException.class, - () -> gatewayMetaState.start(null, null, null, null, null, null, null, List.of(duplicatePlugin, duplicatePlugin)) + () -> gatewayMetaState.start( + null, + null, + null, + null, + null, + null, + null, + List.of(duplicatePlugin, duplicatePlugin), + CompatibilityVersionsUtils.staticCurrent() + ) ).getMessage(), containsString("multiple persisted-state factories") ); @@ -173,7 +184,7 @@ public Optional getPersistedStateFactory() { public Optional getPersistedStateFactory() { return Optional.of((settings, transportService, persistedClusterStateService) -> testPersistedState); } - })); + }), CompatibilityVersionsUtils.staticCurrent()); assertSame(testPersistedState, gatewayMetaState.getPersistedState()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java b/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java index 9aea805b55b30..42fb21d13739f 100644 --- a/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java +++ b/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java @@ -8,13 +8,14 @@ package org.elasticsearch.gateway; -import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadataVerifier; import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.version.CompatibilityVersions; +import org.elasticsearch.cluster.version.CompatibilityVersionsUtils; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; @@ -50,9 +51,14 @@ Metadata upgradeMetadataForNode(Metadata metadata, IndexMetadataVerifier indexMe } @Override - ClusterState prepareInitialClusterState(TransportService transportService, ClusterService clusterService, ClusterState clusterState) { + ClusterState prepareInitialClusterState( + TransportService transportService, + ClusterService clusterService, + ClusterState clusterState, + CompatibilityVersions compatibilityVersions + ) { // Just set localNode here, not to mess with ClusterService and IndicesService mocking - return ClusterStateUpdaters.setLocalNode(clusterState, localNode, TransportVersion.current()); + return ClusterStateUpdaters.setLocalNode(clusterState, localNode, compatibilityVersions); } public void start(Settings settings, NodeEnvironment nodeEnvironment, NamedXContentRegistry xContentRegistry) { @@ -81,7 +87,8 @@ public void start(Settings settings, NodeEnvironment nodeEnvironment, NamedXCont new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L ), - List.of() + List.of(), + CompatibilityVersionsUtils.staticCurrent() ); } } From 1c12f68d14b743d9e2344e0406ab755d2e6d2407 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 8 Sep 2023 14:13:24 +0100 Subject: [PATCH 35/69] Remove unused wrapWithTimeout override (#99360) This was deprecated in #99334 and is now unused so can be removed. --- .../action/support/ListenerTimeouts.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java b/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java index a4c66f6e0b541..a4e2121c56aaf 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java +++ b/server/src/main/java/org/elasticsearch/action/support/ListenerTimeouts.java @@ -68,22 +68,6 @@ public static ActionListener wrapWithTimeout( return wrappedListener; } - /** - * @deprecated Use {@link #wrapWithTimeout(ThreadPool, TimeValue, Executor, ActionListener, Consumer)} instead. - */ - @Deprecated(forRemoval = true) - public static ActionListener wrapWithTimeout( - ThreadPool threadPool, - TimeValue timeout, - String executor, - ActionListener listener, - Consumer> onTimeout - ) { - TimeoutableListener wrappedListener = new TimeoutableListener<>(listener, onTimeout); - wrappedListener.cancellable = threadPool.schedule(wrappedListener, timeout, executor); - return wrappedListener; - } - private static class TimeoutableListener implements ActionListener, Runnable { private final AtomicBoolean isDone = new AtomicBoolean(false); From 648d30ba2a08cb36d5444d1b72edbe26a62aa12f Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Fri, 8 Sep 2023 16:15:07 +0200 Subject: [PATCH 36/69] Fix sparse vector tests (#99339) * Fix sparse vector error so it can catch errors thrown before 8.11 --- .../test/search.vectors/90_sparse_vector.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml index eb967d6d662c0..2ddb95d6fc139 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml @@ -122,12 +122,10 @@ --- "Sparse vector in 8.x": - skip: - # version: " - 7.99.99, 8.11.0 - " - # reason: "sparse_vector field type not supported in 8.x until 8.11.0" - version: "all" - reason: "https://github.com/elastic/elasticsearch/issues/99318" + version: " - 7.99.99, 8.11.0 - " + reason: "sparse_vector field type not supported in 8.x until 8.11.0" - do: - catch: /The \[sparse_vector\] field type is no longer supported/ + catch: /The \[sparse_vector\] field type .* supported/ indices.create: index: test body: From 7d8615e20a2fdbf643dd4b53efc5f19eb9767c15 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Fri, 8 Sep 2023 16:49:40 +0200 Subject: [PATCH 37/69] Fix non-deterministic ESQL spec tests (#99356) The test was non-deterministic because the result did not have an explicit order (ESQL does not enforce any specific order in this case, so the result can change depending on how the query is executed, eg. in multi-node). Adding option to ignore the order of results. --- .../src/main/resources/topN.csv-spec | 44 +++++++++---------- .../src/main/resources/version.csv-spec | 1 + 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec index c8d209f932ad6..e09bc933340d1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/topN.csv-spec @@ -42,29 +42,29 @@ epsilon |eth2 |[fe81::cae2:65ff:fece:feb9, fe82::cae2:65ff:fece ; complexMultiSortingFields_SameFieldAscAndDesc -from employees | sort job_positions asc, languages nulls first, gender desc, job_positions desc | limit 20 | keep job_positions, languages, gender; +from employees | sort job_positions asc, languages nulls first, gender desc, job_positions desc, emp_no | limit 20 | keep job_positions, languages, gender, emp_no; - job_positions:keyword |languages:integer|gender:keyword -[Accountant, Internship, Python Developer, Tech Lead] |null |M -Accountant |null |M -[Accountant, Purchase Manager, Support Engineer] |null |F -[Accountant, Business Analyst, Data Scientist, Python Developer] |1 |M -[Accountant, Principal Support Engineer, Support Engineer, Tech Lead] |1 |F -[Accountant, Junior Developer] |1 |F -[Accountant, Purchase Manager, Python Developer, Reporting Analyst] |2 |null -[Accountant, Senior Python Developer, Tech Lead] |2 |M -[Accountant, Junior Developer, Reporting Analyst, Support Engineer] |2 |M -[Accountant, Senior Python Developer] |2 |M -[Accountant, Internship] |2 |M -[Accountant, Junior Developer, Principal Support Engineer, Purchase Manager]|3 |M -[Accountant, Business Analyst, Purchase Manager] |3 |M -[Accountant, Junior Developer] |4 |F -[Accountant, Head Human Resources] |5 |null -[Accountant, Business Analyst, Senior Python Developer, Tech Lead] |5 |M -[Accountant, Data Scientist, Internship, Senior Python Developer] |5 |M -[Accountant, Principal Support Engineer, Senior Python Developer] |5 |F -[Architect, Principal Support Engineer, Purchase Manager, Senior Team Lead] |2 |F -[Architect, Internship, Principal Support Engineer] |2 |F + job_positions:keyword |languages:integer|gender:keyword | emp_no:integer +[Accountant, Internship, Python Developer, Tech Lead] |null |M |10028 +Accountant |null |M |10025 +[Accountant, Purchase Manager, Support Engineer] |null |F |10023 +[Accountant, Business Analyst, Data Scientist, Python Developer] |1 |M |10034 +[Accountant, Principal Support Engineer, Support Engineer, Tech Lead] |1 |F |10044 +[Accountant, Junior Developer] |1 |F |10092 +[Accountant, Purchase Manager, Python Developer, Reporting Analyst] |2 |null |10016 +[Accountant, Senior Python Developer, Tech Lead] |2 |M |10037 +[Accountant, Junior Developer, Reporting Analyst, Support Engineer] |2 |M |10050 +[Accountant, Senior Python Developer] |2 |M |10001 +[Accountant, Internship] |2 |M |10081 +[Accountant, Junior Developer, Principal Support Engineer, Purchase Manager]|3 |M |10045 +[Accountant, Business Analyst, Purchase Manager] |3 |M |10051 +[Accountant, Junior Developer] |4 |F |10089 +[Accountant, Head Human Resources] |5 |null |10012 +[Accountant, Business Analyst, Senior Python Developer, Tech Lead] |5 |M |10085 +[Accountant, Data Scientist, Internship, Senior Python Developer] |5 |M |10066 +[Accountant, Principal Support Engineer, Senior Python Developer] |5 |F |10094 +[Architect, Principal Support Engineer, Purchase Manager, Senior Team Lead] |2 |F |10059 +[Architect, Internship, Principal Support Engineer] |2 |F |10078 ; sortingOnMVDoubles diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec index b7b3ca1c99d3c..df1fa6e67f279 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec @@ -6,6 +6,7 @@ selectAll FROM apps; +ignoreOrder:true id:integer |name:keyword |version:version 1 |aaaaa |1 From 7dcfdd6215ee08a434869b76f86e57f66b923ef4 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 8 Sep 2023 11:10:41 -0400 Subject: [PATCH 38/69] [buildkite] Pause periodic triggers temporarily due to agent capacity (#99373) --- .buildkite/scripts/periodic.trigger.sh | 2 + catalog-info.yaml | 78 +++++++++++++------------- 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/.buildkite/scripts/periodic.trigger.sh b/.buildkite/scripts/periodic.trigger.sh index 3571d112c5b6d..36d106e87ee9c 100644 --- a/.buildkite/scripts/periodic.trigger.sh +++ b/.buildkite/scripts/periodic.trigger.sh @@ -2,6 +2,8 @@ set -euo pipefail +exit 0 + echo "steps:" source .buildkite/scripts/branches.sh diff --git a/catalog-info.yaml b/catalog-info.yaml index b9c0c98b45e6c..4eb88fd121985 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -98,45 +98,45 @@ spec: build_pull_requests: false publish_commit_status: false trigger_mode: none ---- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json -apiVersion: backstage.io/v1alpha1 -kind: Resource -metadata: - name: buildkite-pipeline-elasticsearch-periodic-trigger - description: Triggers periodic pipelines for all required branches - links: - - title: Pipeline - url: https://buildkite.com/elastic/elasticsearch-periodic-trigger -spec: - type: buildkite-pipeline - system: buildkite - owner: group:elasticsearch-team - implementation: - apiVersion: buildkite.elastic.dev/v1 - kind: Pipeline - metadata: - description: ":elasticsearch: Triggers periodic pipelines for all required branches" - name: elasticsearch / periodic / trigger - spec: - repository: elastic/elasticsearch - pipeline_file: .buildkite/scripts/periodic.trigger.sh - branch_configuration: main - teams: - elasticsearch-team: {} - ml-core: {} - everyone: - access_level: BUILD_AND_READ - provider_settings: - build_branches: false - build_pull_requests: false - publish_commit_status: false - trigger_mode: none - schedules: - Periodically on main: - branch: main - cronline: "0 0,8,16 * * * America/New_York" - message: "Triggers pipelines 3x daily" +# --- +# # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +# apiVersion: backstage.io/v1alpha1 +# kind: Resource +# metadata: +# name: buildkite-pipeline-elasticsearch-periodic-trigger +# description: Triggers periodic pipelines for all required branches +# links: +# - title: Pipeline +# url: https://buildkite.com/elastic/elasticsearch-periodic-trigger +# spec: +# type: buildkite-pipeline +# system: buildkite +# owner: group:elasticsearch-team +# implementation: +# apiVersion: buildkite.elastic.dev/v1 +# kind: Pipeline +# metadata: +# description: ":elasticsearch: Triggers periodic pipelines for all required branches" +# name: elasticsearch / periodic / trigger +# spec: +# repository: elastic/elasticsearch +# pipeline_file: .buildkite/scripts/periodic.trigger.sh +# branch_configuration: main +# teams: +# elasticsearch-team: {} +# ml-core: {} +# everyone: +# access_level: BUILD_AND_READ +# provider_settings: +# build_branches: false +# build_pull_requests: false +# publish_commit_status: false +# trigger_mode: none +# schedules: +# Periodically on main: +# branch: main +# cronline: "0 0,8,16 * * * America/New_York" +# message: "Triggers pipelines 3x daily" --- # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json apiVersion: backstage.io/v1alpha1 From 04381664c1b2bb89f2b7ecb699b1b20051f31fc5 Mon Sep 17 00:00:00 2001 From: dreamquster Date: Fri, 8 Sep 2023 23:27:59 +0800 Subject: [PATCH 39/69] ESQL: Implement 'right' function (#98974) Add the 'right' function, which extracts a substring beginning from its right end (opposite function of 'left'). --------- Co-authored-by: Alexander Spies --- docs/changelog/98974.yaml | 5 + docs/reference/esql/esql-functions.asciidoc | 10 +- docs/reference/esql/functions/left.asciidoc | 9 +- docs/reference/esql/functions/right.asciidoc | 19 ++ .../esql/functions/signature/left.svg | 2 +- .../esql/functions/signature/right.svg | 1 + .../esql/functions/types/left.asciidoc | 2 +- .../esql/functions/types/right.asciidoc | 5 + .../src/main/resources/show.csv-spec | 3 +- .../src/main/resources/string.csv-spec | 20 ++ .../function/scalar/string/LeftEvaluator.java | 36 +-- .../scalar/string/RightEvaluator.java | 93 ++++++++ .../function/EsqlFunctionRegistry.java | 2 + .../function/scalar/string/Left.java | 18 +- .../function/scalar/string/Right.java | 137 ++++++++++++ .../xpack/esql/io/stream/PlanNamedTypes.java | 13 ++ .../function/scalar/string/LeftTests.java | 133 +++++++++-- .../function/scalar/string/RightTests.java | 210 ++++++++++++++++++ 18 files changed, 658 insertions(+), 60 deletions(-) create mode 100644 docs/changelog/98974.yaml create mode 100644 docs/reference/esql/functions/right.asciidoc create mode 100644 docs/reference/esql/functions/signature/right.svg create mode 100644 docs/reference/esql/functions/types/right.asciidoc create mode 100644 x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java diff --git a/docs/changelog/98974.yaml b/docs/changelog/98974.yaml new file mode 100644 index 0000000000000..90950986141ab --- /dev/null +++ b/docs/changelog/98974.yaml @@ -0,0 +1,5 @@ +pr: 98974 +summary: "ESQL: RIGHT function" +area: ES|QL +type: feature +issues: [] diff --git a/docs/reference/esql/esql-functions.asciidoc b/docs/reference/esql/esql-functions.asciidoc index 49b0be636d0a0..1e29d312906ae 100644 --- a/docs/reference/esql/esql-functions.asciidoc +++ b/docs/reference/esql/esql-functions.asciidoc @@ -32,10 +32,10 @@ these functions: * <> * <> * <> +* <> * <> * <> * <> -* <> * <> * <> * <> @@ -47,13 +47,14 @@ these functions: * <> * <> * <> +* <> * <> +* <> * <> * <> * <> * <> * <> -* <> * <> * <> * <> @@ -94,10 +95,10 @@ include::functions/is_finite.asciidoc[] include::functions/is_infinite.asciidoc[] include::functions/is_nan.asciidoc[] include::functions/least.asciidoc[] +include::functions/left.asciidoc[] include::functions/length.asciidoc[] include::functions/log10.asciidoc[] include::functions/ltrim.asciidoc[] -include::functions/rtrim.asciidoc[] include::functions/mv_avg.asciidoc[] include::functions/mv_concat.asciidoc[] include::functions/mv_count.asciidoc[] @@ -109,14 +110,15 @@ include::functions/mv_sum.asciidoc[] include::functions/now.asciidoc[] include::functions/pi.asciidoc[] include::functions/pow.asciidoc[] +include::functions/right.asciidoc[] include::functions/round.asciidoc[] +include::functions/rtrim.asciidoc[] include::functions/sin.asciidoc[] include::functions/sinh.asciidoc[] include::functions/split.asciidoc[] include::functions/sqrt.asciidoc[] include::functions/starts_with.asciidoc[] include::functions/substring.asciidoc[] -include::functions/left.asciidoc[] include::functions/tan.asciidoc[] include::functions/tanh.asciidoc[] include::functions/tau.asciidoc[] diff --git a/docs/reference/esql/functions/left.asciidoc b/docs/reference/esql/functions/left.asciidoc index 42537d9560ebf..91a2d544b279f 100644 --- a/docs/reference/esql/functions/left.asciidoc +++ b/docs/reference/esql/functions/left.asciidoc @@ -1,8 +1,9 @@ [[esql-left]] === `LEFT` +[.text-center] +image::esql/functions/signature/left.svg[Embedded,opts=inline] -Return the substring that extract 'length' chars -from string starting from 0. +Return the substring that extracts 'length' chars from the 'string' starting from the left. [source.merge.styled,esql] ---- @@ -12,3 +13,7 @@ include::{esql-specs}/string.csv-spec[tag=left] |=== include::{esql-specs}/string.csv-spec[tag=left-result] |=== + +Supported types: + +include::types/left.asciidoc[] diff --git a/docs/reference/esql/functions/right.asciidoc b/docs/reference/esql/functions/right.asciidoc new file mode 100644 index 0000000000000..ec55a2affbe40 --- /dev/null +++ b/docs/reference/esql/functions/right.asciidoc @@ -0,0 +1,19 @@ +[[esql-right]] +=== `RIGHT` +[.text-center] +image::esql/functions/signature/right.svg[Embedded,opts=inline] + +Return the substring that extracts 'length' chars from the 'string' starting from the right. + +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=right] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/string.csv-spec[tag=right-result] +|=== + +Supported types: + +include::types/right.asciidoc[] diff --git a/docs/reference/esql/functions/signature/left.svg b/docs/reference/esql/functions/signature/left.svg index e04d4cc606fac..ec14bf8c72131 100644 --- a/docs/reference/esql/functions/signature/left.svg +++ b/docs/reference/esql/functions/signature/left.svg @@ -1 +1 @@ -LEFT(arg1,arg2) \ No newline at end of file +LEFT(string,length) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/right.svg b/docs/reference/esql/functions/signature/right.svg new file mode 100644 index 0000000000000..0afa5dbf01f16 --- /dev/null +++ b/docs/reference/esql/functions/signature/right.svg @@ -0,0 +1 @@ +RIGHT(string,length) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/left.asciidoc b/docs/reference/esql/functions/types/left.asciidoc index 180f3094f7ef6..c30a055f3be49 100644 --- a/docs/reference/esql/functions/types/left.asciidoc +++ b/docs/reference/esql/functions/types/left.asciidoc @@ -1,5 +1,5 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== -arg1 | arg2 | result +string | length | result keyword | integer | keyword |=== diff --git a/docs/reference/esql/functions/types/right.asciidoc b/docs/reference/esql/functions/types/right.asciidoc new file mode 100644 index 0000000000000..c30a055f3be49 --- /dev/null +++ b/docs/reference/esql/functions/types/right.asciidoc @@ -0,0 +1,5 @@ +[%header.monospaced.styled,format=dsv,separator=|] +|=== +string | length | result +keyword | integer | keyword +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec index c684e93cb986f..efd1e3591e2b8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec @@ -36,7 +36,7 @@ is_finite |is_finite(arg1) is_infinite |is_infinite(arg1) is_nan |is_nan(arg1) least |least(first, rest...) -left |left(arg1, arg2) +left |left(string, length) length |length(arg1) log10 |log10(n) ltrim |ltrim(arg1) @@ -56,6 +56,7 @@ now |now() percentile |percentile(arg1, arg2) pi |pi() pow |pow(base, exponent) +right |right(string, length) round |round(arg1, arg2) rtrim |rtrim(arg1) sin |sin(n) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index 8a030e7465438..aa893e63e1a30 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -733,3 +733,23 @@ Bamford |Bam Bernatsky |Ber // end::left-result[] ; + +right +// tag::right[] +FROM employees +| KEEP last_name +| EVAL right = RIGHT(last_name, 3) +| SORT last_name ASC +| LIMIT 5 +// end::right[] +; + +// tag::right-result[] +last_name:keyword | right:keyword +Awdeh |deh +Azuma |uma +Baek |aek +Bamford |ord +Bernatsky |sky +// end::right-result[] +; diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java index e12d988d8b5ab..23ccb4d544331 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java @@ -4,10 +4,10 @@ // 2.0. package org.elasticsearch.xpack.esql.expression.function.scalar.string; -import java.lang.IllegalArgumentException; import java.lang.Override; import java.lang.String; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.BytesRefVector; @@ -15,26 +15,24 @@ import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.xpack.esql.expression.function.Warnings; -import org.elasticsearch.xpack.ql.tree.Source; /** * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Left}. * This class is generated. Do not edit it. */ public final class LeftEvaluator implements EvalOperator.ExpressionEvaluator { - private final Warnings warnings; - private final BytesRef out; + private final UnicodeUtil.UTF8CodePoint cp; + private final EvalOperator.ExpressionEvaluator str; private final EvalOperator.ExpressionEvaluator length; - public LeftEvaluator(Source source, BytesRef out, EvalOperator.ExpressionEvaluator str, - EvalOperator.ExpressionEvaluator length) { - this.warnings = new Warnings(source); + public LeftEvaluator(BytesRef out, UnicodeUtil.UTF8CodePoint cp, + EvalOperator.ExpressionEvaluator str, EvalOperator.ExpressionEvaluator length) { this.out = out; + this.cp = cp; this.str = str; this.length = length; } @@ -59,7 +57,7 @@ public Block eval(Page page) { if (lengthVector == null) { return eval(page.getPositionCount(), strBlock, lengthBlock); } - return eval(page.getPositionCount(), strVector, lengthVector); + return eval(page.getPositionCount(), strVector, lengthVector).asBlock(); } public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock, IntBlock lengthBlock) { @@ -74,32 +72,22 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock, IntBlock le result.appendNull(); continue position; } - try { - result.appendBytesRef(Left.process(out, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), lengthBlock.getInt(lengthBlock.getFirstValueIndex(p)))); - } catch (IllegalArgumentException e) { - warnings.registerException(e); - result.appendNull(); - } + result.appendBytesRef(Left.process(out, cp, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), lengthBlock.getInt(lengthBlock.getFirstValueIndex(p)))); } return result.build(); } - public BytesRefBlock eval(int positionCount, BytesRefVector strVector, IntVector lengthVector) { - BytesRefBlock.Builder result = BytesRefBlock.newBlockBuilder(positionCount); + public BytesRefVector eval(int positionCount, BytesRefVector strVector, IntVector lengthVector) { + BytesRefVector.Builder result = BytesRefVector.newVectorBuilder(positionCount); BytesRef strScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { - try { - result.appendBytesRef(Left.process(out, strVector.getBytesRef(p, strScratch), lengthVector.getInt(p))); - } catch (IllegalArgumentException e) { - warnings.registerException(e); - result.appendNull(); - } + result.appendBytesRef(Left.process(out, cp, strVector.getBytesRef(p, strScratch), lengthVector.getInt(p))); } return result.build(); } @Override public String toString() { - return "LeftEvaluator[" + "out=" + out + ", str=" + str + ", length=" + length + "]"; + return "LeftEvaluator[" + "str=" + str + ", length=" + length + "]"; } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java new file mode 100644 index 0000000000000..f075c37d05dd9 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java @@ -0,0 +1,93 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.EvalOperator; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Right}. + * This class is generated. Do not edit it. + */ +public final class RightEvaluator implements EvalOperator.ExpressionEvaluator { + private final BytesRef out; + + private final UnicodeUtil.UTF8CodePoint cp; + + private final EvalOperator.ExpressionEvaluator str; + + private final EvalOperator.ExpressionEvaluator length; + + public RightEvaluator(BytesRef out, UnicodeUtil.UTF8CodePoint cp, + EvalOperator.ExpressionEvaluator str, EvalOperator.ExpressionEvaluator length) { + this.out = out; + this.cp = cp; + this.str = str; + this.length = length; + } + + @Override + public Block eval(Page page) { + Block strUncastBlock = str.eval(page); + if (strUncastBlock.areAllValuesNull()) { + return Block.constantNullBlock(page.getPositionCount()); + } + BytesRefBlock strBlock = (BytesRefBlock) strUncastBlock; + Block lengthUncastBlock = length.eval(page); + if (lengthUncastBlock.areAllValuesNull()) { + return Block.constantNullBlock(page.getPositionCount()); + } + IntBlock lengthBlock = (IntBlock) lengthUncastBlock; + BytesRefVector strVector = strBlock.asVector(); + if (strVector == null) { + return eval(page.getPositionCount(), strBlock, lengthBlock); + } + IntVector lengthVector = lengthBlock.asVector(); + if (lengthVector == null) { + return eval(page.getPositionCount(), strBlock, lengthBlock); + } + return eval(page.getPositionCount(), strVector, lengthVector).asBlock(); + } + + public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock, IntBlock lengthBlock) { + BytesRefBlock.Builder result = BytesRefBlock.newBlockBuilder(positionCount); + BytesRef strScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (strBlock.isNull(p) || strBlock.getValueCount(p) != 1) { + result.appendNull(); + continue position; + } + if (lengthBlock.isNull(p) || lengthBlock.getValueCount(p) != 1) { + result.appendNull(); + continue position; + } + result.appendBytesRef(Right.process(out, cp, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), lengthBlock.getInt(lengthBlock.getFirstValueIndex(p)))); + } + return result.build(); + } + + public BytesRefVector eval(int positionCount, BytesRefVector strVector, IntVector lengthVector) { + BytesRefVector.Builder result = BytesRefVector.newVectorBuilder(positionCount); + BytesRef strScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + result.appendBytesRef(Right.process(out, cp, strVector.getBytesRef(p, strScratch), lengthVector.getInt(p))); + } + return result.build(); + } + + @Override + public String toString() { + return "RightEvaluator[" + "str=" + str + ", length=" + length + "]"; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index de4748537efa2..d4029fd8ac4f0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -74,6 +74,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; @@ -143,6 +144,7 @@ private FunctionDefinition[][] functions() { def(RTrim.class, RTrim::new, "rtrim"), def(Trim.class, Trim::new, "trim"), def(Left.class, Left::new, "left"), + def(Right.class, Right::new, "right"), def(StartsWith.class, StartsWith::new, "starts_with") }, // date new FunctionDefinition[] { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java index 9c1133769f846..121225765f5af 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.Named; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; @@ -32,7 +33,7 @@ import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString; /** - * left(foo, len) is a alias that substring(foo, 0, len) + * {code left(foo, len)} is an alias to {code substring(foo, 0, len)} */ public class Left extends ScalarFunction implements EvaluatorMapper { @@ -42,20 +43,24 @@ public class Left extends ScalarFunction implements EvaluatorMapper { private final Expression length; - public Left(Source source, Expression str, Expression length) { + public Left(Source source, @Named("string") Expression str, @Named("length") Expression length) { super(source, Arrays.asList(str, length)); this.source = source; this.str = str; this.length = length; } - @Evaluator(warnExceptions = IllegalArgumentException.class) - static BytesRef process(@Fixed BytesRef out, BytesRef str, int length) { + @Evaluator + static BytesRef process( + @Fixed(includeInToString = false) BytesRef out, + @Fixed(includeInToString = false) UnicodeUtil.UTF8CodePoint cp, + BytesRef str, + int length + ) { out.bytes = str.bytes; out.offset = str.offset; out.length = str.length; int curLenStart = 0; - UnicodeUtil.UTF8CodePoint cp = new UnicodeUtil.UTF8CodePoint(); for (int i = 0; i < length && curLenStart < out.length; i++, curLenStart += cp.numBytes) { UnicodeUtil.codePointAt(out.bytes, out.offset + curLenStart, cp); } @@ -72,7 +77,8 @@ public Supplier toEvaluator( Supplier lengthSupplier = toEvaluator.apply(length); return () -> { BytesRef out = new BytesRef(); - return new LeftEvaluator(source, out, strSupplier.get(), lengthSupplier.get()); + UnicodeUtil.UTF8CodePoint cp = new UnicodeUtil.UTF8CodePoint(); + return new LeftEvaluator(out, cp, strSupplier.get(), lengthSupplier.get()); }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java new file mode 100644 index 0000000000000..e41fb21b57797 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.ann.Fixed; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.Named; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isInteger; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString; + +/** + * {code right(foo, len)} is an alias to {code substring(foo, foo.length-len, len)} + */ +public class Right extends ScalarFunction implements EvaluatorMapper { + + private final Source source; + + private final Expression str; + + private final Expression length; + + public Right(Source source, @Named("string") Expression str, @Named("length") Expression length) { + super(source, Arrays.asList(str, length)); + this.source = source; + this.str = str; + this.length = length; + } + + @Evaluator + static BytesRef process( + @Fixed(includeInToString = false) BytesRef out, + @Fixed(includeInToString = false) UnicodeUtil.UTF8CodePoint cp, + BytesRef str, + int length + ) { + out.bytes = str.bytes; + out.offset = str.offset; + out.length = str.length; + int codeLen = UnicodeUtil.codePointCount(str); + // skip the first skipLen codePoint + int skipLen = Math.max(codeLen - length, 0); + int endOffset = str.offset + str.length; + for (int i = 0; i < skipLen && out.offset < endOffset; i++) { + UnicodeUtil.codePointAt(out.bytes, out.offset, cp); + out.offset += cp.numBytes; + out.length -= cp.numBytes; + } + return out; + } + + @Override + public Supplier toEvaluator( + Function> toEvaluator + ) { + + Supplier strSupplier = toEvaluator.apply(str); + Supplier lengthSupplier = toEvaluator.apply(length); + return () -> { + BytesRef out = new BytesRef(); + UnicodeUtil.UTF8CodePoint cp = new UnicodeUtil.UTF8CodePoint(); + return new RightEvaluator(out, cp, strSupplier.get(), lengthSupplier.get()); + }; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Right(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Right::new, str, length); + } + + @Override + public DataType dataType() { + return DataTypes.KEYWORD; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution resolution = isString(str, sourceText(), FIRST); + if (resolution.unresolved()) { + return resolution; + } + + resolution = isInteger(length, sourceText(), SECOND); + if (resolution.unresolved()) { + return resolution; + } + + return TypeResolution.TYPE_RESOLVED; + } + + @Override + public boolean foldable() { + return str.foldable() && length.foldable(); + } + + @Override + public Object fold() { + return EvaluatorMapper.super.fold(); + } + + @Override + public ScriptTemplate asScript() { + throw new UnsupportedOperationException(); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index eb8b3777ee658..edc8bcacda9d4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -92,6 +92,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; @@ -346,6 +347,7 @@ public static List namedTypeEntries() { of(ScalarFunction.class, StartsWith.class, PlanNamedTypes::writeStartsWith, PlanNamedTypes::readStartsWith), of(ScalarFunction.class, Substring.class, PlanNamedTypes::writeSubstring, PlanNamedTypes::readSubstring), of(ScalarFunction.class, Left.class, PlanNamedTypes::writeLeft, PlanNamedTypes::readLeft), + of(ScalarFunction.class, Right.class, PlanNamedTypes::writeRight, PlanNamedTypes::readRight), of(ScalarFunction.class, Split.class, PlanNamedTypes::writeSplit, PlanNamedTypes::readSplit), of(ScalarFunction.class, Tau.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar), // ArithmeticOperations @@ -1304,6 +1306,17 @@ static void writeLeft(PlanStreamOutput out, Left left) throws IOException { out.writeExpression(fields.get(1)); } + static Right readRight(PlanStreamInput in) throws IOException { + return new Right(Source.EMPTY, in.readExpression(), in.readExpression()); + } + + static void writeRight(PlanStreamOutput out, Right right) throws IOException { + List fields = right.children(); + assert fields.size() == 2; + out.writeExpression(fields.get(0)); + out.writeExpression(fields.get(1)); + } + static Split readSplit(PlanStreamInput in) throws IOException { return new Split(Source.EMPTY, in.readExpression(), in.readExpression()); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java index fbbd4912b6bb0..eb68509fdfafa 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java @@ -36,35 +36,138 @@ public LeftTests(@Name("TestCase") Supplier testCaseS @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - suppliers.add(new TestCaseSupplier("long", () -> { - int length = between(1, 10); - String text = randomAlphaOfLength(10); + + suppliers.add(new TestCaseSupplier("empty string", () -> { + int length = between(-64, 64); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(""), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef("")) + ); + })); + + suppliers.add(new TestCaseSupplier("ascii", () -> { + String text = randomAlphaOfLengthBetween(1, 64); + int length = between(1, text.length()); return new TestCaseSupplier.TestCase( List.of( new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") ), - "LeftEvaluator[out=[], str=Attribute[channel=0], length=Attribute[channel=1]]", + "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", DataTypes.KEYWORD, - equalTo(new BytesRef(text.substring(0, length))) + equalTo(new BytesRef(unicodeLeftSubstring(text, length))) ); })); - suppliers.add(new TestCaseSupplier("short", () -> { - int length = between(2, 10); - String text = randomAlphaOfLength(1); + suppliers.add(new TestCaseSupplier("ascii longer than string", () -> { + String text = randomAlphaOfLengthBetween(1, 64); + int length = between(text.length(), 128); return new TestCaseSupplier.TestCase( List.of( new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") ), - "LeftEvaluator[out=[], str=Attribute[channel=0], length=Attribute[channel=1]]", + "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", DataTypes.KEYWORD, equalTo(new BytesRef(text)) ); })); + suppliers.add(new TestCaseSupplier("ascii zero length", () -> { + String text = randomAlphaOfLengthBetween(1, 64); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(0, DataTypes.INTEGER, "length") + ), + "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef("")) + ); + })); + suppliers.add(new TestCaseSupplier("ascii negative length", () -> { + String text = randomAlphaOfLengthBetween(1, 64); + int length = between(-128, -1); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef("")) + ); + })); + + suppliers.add(new TestCaseSupplier("unicode", () -> { + String text = randomUnicodeOfLengthBetween(1, 64); + int length = between(1, text.length()); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef(unicodeLeftSubstring(text, length))) + ); + })); + suppliers.add(new TestCaseSupplier("unicode longer than string", () -> { + String text = randomUnicodeOfLengthBetween(1, 64); + int length = between(text.length(), 128); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef(text)) + ); + })); + suppliers.add(new TestCaseSupplier("unicode zero length", () -> { + String text = randomUnicodeOfLengthBetween(1, 64); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(0, DataTypes.INTEGER, "length") + ), + "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef("")) + ); + })); + suppliers.add(new TestCaseSupplier("unicode negative length", () -> { + String text = randomUnicodeOfLengthBetween(1, 64); + int length = between(-128, -1); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef("")) + ); + })); + return parameterSuppliersFromTypedData(suppliers); } + private static String unicodeLeftSubstring(String str, int length) { + if (length < 0) { + return ""; + } else { + return str.codePoints() + .limit(length) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + } + } + @Override protected Expression build(Source source, List args) { return new Left(source, args.get(0), args.get(1)); @@ -86,18 +189,6 @@ public Matcher resultsMatcher(List typedData return equalTo(new BytesRef(str.substring(0, length))); } - public void testReasonableLength() { - assertThat(process("a fox call", 5), equalTo("a fox")); - } - - public void testMassiveLength() { - assertThat(process("a fox call", 10), equalTo("a fox call")); - } - - public void testNegativeLength() { - assertThat(process("a fox call", -1), equalTo("")); - } - public void testUnicode() { final String s = "a\ud83c\udf09tiger"; assert s.codePointCount(0, s.length()) == 7; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java new file mode 100644 index 0000000000000..540051d9ac8b8 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java @@ -0,0 +1,210 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Literal; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.hamcrest.Matcher; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; +import static org.hamcrest.Matchers.equalTo; + +public class RightTests extends AbstractScalarFunctionTestCase { + public RightTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = new ArrayList<>(); + + suppliers.add(new TestCaseSupplier("empty string", () -> { + int length = between(-64, 64); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(""), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef("")) + ); + })); + + suppliers.add(new TestCaseSupplier("ascii", () -> { + String text = randomAlphaOfLengthBetween(1, 64); + int length = between(1, text.length()); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef(unicodeRightSubstring(text, length))) + ); + })); + suppliers.add(new TestCaseSupplier("ascii longer than string", () -> { + String text = randomAlphaOfLengthBetween(1, 64); + int length = between(text.length(), 128); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef(text)) + ); + })); + suppliers.add(new TestCaseSupplier("ascii zero length", () -> { + String text = randomAlphaOfLengthBetween(1, 64); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(0, DataTypes.INTEGER, "length") + ), + "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef("")) + ); + })); + suppliers.add(new TestCaseSupplier("ascii negative length", () -> { + String text = randomAlphaOfLengthBetween(1, 64); + int length = between(-128, -1); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef("")) + ); + })); + + suppliers.add(new TestCaseSupplier("unicode", () -> { + String text = randomUnicodeOfLengthBetween(1, 64); + int length = between(1, text.length()); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef(unicodeRightSubstring(text, length))) + ); + })); + suppliers.add(new TestCaseSupplier("unicode longer than string", () -> { + String text = randomUnicodeOfLengthBetween(1, 64); + int length = between(text.length(), 128); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef(text)) + ); + })); + suppliers.add(new TestCaseSupplier("unicode zero length", () -> { + String text = randomUnicodeOfLengthBetween(1, 64); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(0, DataTypes.INTEGER, "length") + ), + "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef("")) + ); + })); + suppliers.add(new TestCaseSupplier("unicode negative length", () -> { + String text = randomUnicodeOfLengthBetween(1, 64); + int length = between(-128, -1); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"), + new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length") + ), + "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataTypes.KEYWORD, + equalTo(new BytesRef("")) + ); + })); + + return parameterSuppliersFromTypedData(suppliers); + } + + private static String unicodeRightSubstring(String str, int length) { + int codepointCount = str.codePointCount(0, str.length()); + int codePointsToSkip = codepointCount - length; + if (codePointsToSkip < 0) { + return str; + } else { + return str.codePoints() + .skip(codePointsToSkip) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + } + } + + @Override + protected Expression build(Source source, List args) { + return new Right(source, args.get(0), args.get(1)); + } + + @Override + protected List argSpec() { + return List.of(required(strings()), required(integers())); + } + + @Override + protected DataType expectedType(List argTypes) { + return DataTypes.KEYWORD; + } + + public Matcher resultsMatcher(List typedData) { + String str = ((BytesRef) typedData.get(0).data()).utf8ToString(); + int length = (Integer) typedData.get(1).data(); + return equalTo(new BytesRef(str.substring(str.length() - length))); + } + + public void testUnicode() { + final String s = "a\ud83c\udf09tiger"; + assert s.codePointCount(0, s.length()) == 7; + assertThat(process(s, 6), equalTo("\ud83c\udf09tiger")); + } + + private String process(String str, int length) { + Block result = evaluator( + new Right(Source.EMPTY, field("str", DataTypes.KEYWORD), new Literal(Source.EMPTY, length, DataTypes.INTEGER)) + ).get().eval(row(List.of(new BytesRef(str)))); + if (null == result) { + return null; + } + BytesRef resultByteRef = ((BytesRef) toJavaObject(result, 0)); + return resultByteRef == null ? null : resultByteRef.utf8ToString(); + } +} From 28e158f798a5a88c0a7dfa6655ca709582597098 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 8 Sep 2023 16:43:48 +0100 Subject: [PATCH 40/69] Migrate LocalHealthMonitor away from name-based executor (#99362) Replaces the executor name with a proper executor. Relates #99027 and friends --- .../health/node/LocalHealthMonitor.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java b/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java index e1ea7728acf51..8d25233a1d6e5 100644 --- a/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java +++ b/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java @@ -45,6 +45,7 @@ import java.util.Objects; import java.util.Set; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.core.Strings.format; @@ -241,7 +242,7 @@ DiskHealthInfo getLastReportedDiskHealthInfo() { static class Monitoring implements Runnable, Scheduler.Cancellable { private final TimeValue interval; - private final String executor; + private final Executor executor; private final Scheduler scheduler; private final ClusterService clusterService; private final DiskCheck diskCheck; @@ -256,7 +257,7 @@ static class Monitoring implements Runnable, Scheduler.Cancellable { private Monitoring( TimeValue interval, Scheduler scheduler, - String executor, + Executor executor, AtomicReference lastReportedDiskHealthInfo, AtomicReference lastSeenHealthNode, DiskCheck diskCheck, @@ -278,7 +279,7 @@ private Monitoring( */ static Monitoring start( TimeValue interval, - Scheduler scheduler, + ThreadPool threadPool, AtomicReference lastReportedDiskHealthInfo, AtomicReference lastSeenHealthNode, DiskCheck diskCheck, @@ -287,15 +288,15 @@ static Monitoring start( ) { Monitoring monitoring = new Monitoring( interval, - scheduler, - ThreadPool.Names.MANAGEMENT, + threadPool, + threadPool.executor(ThreadPool.Names.MANAGEMENT), lastReportedDiskHealthInfo, lastSeenHealthNode, diskCheck, clusterService, client ); - monitoring.scheduledRun = scheduler.schedule(monitoring, TimeValue.ZERO, monitoring.executor); + monitoring.scheduledRun = threadPool.schedule(monitoring, TimeValue.ZERO, monitoring.executor); return monitoring; } From 1eafca53a1f3e695ad2768a76cac304784c30a65 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 8 Sep 2023 17:45:24 +0200 Subject: [PATCH 41/69] Simplify MapperBuilderContext (#99372) No need for the boolean supplier here, we can make sure we don't use this thing with meta fields at compile time already. This makes reasoning about using these objects during mapper parsing/merging much easier by making them obviously immutable (without the chance of the supplier bringing in any outside state). --- .../index/mapper/MapperBuilderContext.java | 26 +++++-------------- .../index/mapper/MappingParser.java | 3 +-- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java index 9ee88f2cfe11f..55bc15528404d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java @@ -10,38 +10,25 @@ import org.elasticsearch.common.Strings; -import java.util.Objects; -import java.util.function.BooleanSupplier; - /** * Holds context for building Mapper objects from their Builders */ public class MapperBuilderContext { + private static final MapperBuilderContext ROOT_SYNTHETIC = new MapperBuilderContext(null, true); + private static final MapperBuilderContext ROOT_NOT_SYNTHETIC = new MapperBuilderContext(null, false); + /** * The root context, to be used when building a tree of mappers */ public static MapperBuilderContext root(boolean isSourceSynthetic) { - return new MapperBuilderContext(null, () -> isSourceSynthetic); - } - - /** - * A context to use to build metadata fields. - */ - public static MapperBuilderContext forMetadata() { - return new MapperBuilderContext(null, () -> { - throw new UnsupportedOperationException("metadata fields can't check if _source is synthetic"); - }); + return isSourceSynthetic ? ROOT_SYNTHETIC : ROOT_NOT_SYNTHETIC; } private final String path; - private final BooleanSupplier isSourceSynthetic; + private final boolean isSourceSynthetic; MapperBuilderContext(String path, boolean isSourceSynthetic) { - this(Objects.requireNonNull(path), () -> isSourceSynthetic); - } - - private MapperBuilderContext(String path, BooleanSupplier isSourceSynthetic) { this.path = path; this.isSourceSynthetic = isSourceSynthetic; } @@ -69,6 +56,7 @@ public String buildFullName(String name) { * Is the {@code _source} field being reconstructed on the fly? */ public boolean isSourceSynthetic() { - return isSourceSynthetic.getAsBoolean(); + return isSourceSynthetic; } + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index f1081743ef82a..a310d2bf0b0ae 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -120,8 +120,7 @@ private Mapping parse(String type, Map mapping) throws MapperPar } @SuppressWarnings("unchecked") Map fieldNodeMap = (Map) fieldNode; - MetadataFieldMapper metadataFieldMapper = typeParser.parse(fieldName, fieldNodeMap, mappingParserContext) - .build(MapperBuilderContext.forMetadata()); + MetadataFieldMapper metadataFieldMapper = typeParser.parse(fieldName, fieldNodeMap, mappingParserContext).build(); metadataMappers.put(metadataFieldMapper.getClass(), metadataFieldMapper); assert fieldNodeMap.isEmpty(); if (metadataFieldMapper instanceof SourceFieldMapper sfm) { From 69f2cc6facc28715d9f3fc3aed311e237f3587bc Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 8 Sep 2023 17:47:10 +0200 Subject: [PATCH 42/69] Remove index analyzers map from TextFieldMapper (#99365) No need to keep the map as a field, we only use it once when building `MappingLookup` and throw it away right away as well, so just build it when needed. --- .../index/mapper/TextFieldMapper.java | 51 ++++++------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index a082c7b5b658f..6d2c5b1cb71ac 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -450,25 +450,6 @@ private SubFieldInfo buildPhraseInfo(FieldType fieldType, TextFieldType parent) return new SubFieldInfo(parent.name() + FAST_PHRASE_SUFFIX, phraseFieldType, a); } - public Map indexAnalyzers(String name, SubFieldInfo phraseFieldInfo, SubFieldInfo prefixFieldInfo) { - Map analyzers = new HashMap<>(); - NamedAnalyzer main = this.analyzers.getIndexAnalyzer(); - analyzers.put(name, main); - if (phraseFieldInfo != null) { - analyzers.put( - phraseFieldInfo.field, - new NamedAnalyzer(main.name() + "_phrase", AnalyzerScope.INDEX, phraseFieldInfo.analyzer) - ); - } - if (prefixFieldInfo != null) { - analyzers.put( - prefixFieldInfo.field, - new NamedAnalyzer(main.name() + "_prefix", AnalyzerScope.INDEX, prefixFieldInfo.analyzer) - ); - } - return analyzers; - } - @Override public TextFieldMapper build(MapperBuilderContext context) { MultiFields multiFields = multiFieldsBuilder.build(this, context); @@ -488,17 +469,7 @@ public TextFieldMapper build(MapperBuilderContext context) { throw new MapperParsingException("Cannot use reserved field name [" + mapper.name() + "]"); } } - return new TextFieldMapper( - name, - fieldType, - tft, - indexAnalyzers(tft.name(), phraseFieldInfo, prefixFieldInfo), - prefixFieldInfo, - phraseFieldInfo, - multiFields, - copyTo.build(), - this - ); + return new TextFieldMapper(name, fieldType, tft, prefixFieldInfo, phraseFieldInfo, multiFields, copyTo.build(), this); } } @@ -1151,13 +1122,10 @@ public Query existsQuery(SearchExecutionContext context) { private final SubFieldInfo prefixFieldInfo; private final SubFieldInfo phraseFieldInfo; - private final Map indexAnalyzerMap; - protected TextFieldMapper( String simpleName, FieldType fieldType, TextFieldType mappedFieldType, - Map indexAnalyzers, SubFieldInfo prefixFieldInfo, SubFieldInfo phraseFieldInfo, MultiFields multiFields, @@ -1188,12 +1156,25 @@ protected TextFieldMapper( this.freqFilter = builder.freqFilter.getValue(); this.fieldData = builder.fieldData.get(); this.indexPhrases = builder.indexPhrases.getValue(); - this.indexAnalyzerMap = Map.copyOf(indexAnalyzers); } @Override public Map indexAnalyzers() { - return indexAnalyzerMap; + Map analyzersMap = new HashMap<>(); + analyzersMap.put(name(), indexAnalyzer); + if (phraseFieldInfo != null) { + analyzersMap.put( + phraseFieldInfo.field, + new NamedAnalyzer(indexAnalyzer.name() + "_phrase", AnalyzerScope.INDEX, phraseFieldInfo.analyzer) + ); + } + if (prefixFieldInfo != null) { + analyzersMap.put( + prefixFieldInfo.field, + new NamedAnalyzer(indexAnalyzer.name() + "_prefix", AnalyzerScope.INDEX, prefixFieldInfo.analyzer) + ); + } + return analyzersMap; } @Override From f140fa100fc6fa573426db95f155f0f8c36d5420 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 8 Sep 2023 11:50:09 -0400 Subject: [PATCH 43/69] [buildkite] Add snyk-dependency-monitoring pipeline (#99325) --- .buildkite/hooks/pre-command | 5 +++++ .buildkite/pipelines/periodic.template.yml | 10 ++++++++++ .buildkite/pipelines/periodic.yml | 10 ++++++++++ 3 files changed, 25 insertions(+) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 9aa36e7464f01..42af0d79b34da 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -69,3 +69,8 @@ if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then DRA_VAULT_ADDR=https://secrets.elastic.co:8200 export DRA_VAULT_ADDR fi + +if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then + SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/migrated/snyk) + export SNYK_TOKEN +fi diff --git a/.buildkite/pipelines/periodic.template.yml b/.buildkite/pipelines/periodic.template.yml index 3457af88c1f0b..e4f844afc3f41 100644 --- a/.buildkite/pipelines/periodic.template.yml +++ b/.buildkite/pipelines/periodic.template.yml @@ -103,3 +103,13 @@ steps: image: family/elasticsearch-ubuntu-2004 diskSizeGb: 350 machineType: custom-32-98304 + - label: Upload Snyk Dependency Graph + command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH + env: + USE_SNYK_CREDENTIALS: "true" + timeout_in_minutes: 20 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + buildDirectory: /dev/shm/bk diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index da72f059acded..29aec69bf3832 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -1104,3 +1104,13 @@ steps: image: family/elasticsearch-ubuntu-2004 diskSizeGb: 350 machineType: custom-32-98304 + - label: Upload Snyk Dependency Graph + command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH + env: + USE_SNYK_CREDENTIALS: "true" + timeout_in_minutes: 20 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + buildDirectory: /dev/shm/bk From f5c32d8a9c0530251c9aaaa9cde529b4702b437a Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 8 Sep 2023 18:32:03 +0200 Subject: [PATCH 44/69] ESQL: Log execution time consistently (#99286) Use a StopWatch to make the logged query execution time consistent with the execution time sent to the user. --- docs/changelog/99286.yaml | 5 ++ .../esql/action/EsqlResponseListener.java | 71 ++++++++++++++++--- 2 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 docs/changelog/99286.yaml diff --git a/docs/changelog/99286.yaml b/docs/changelog/99286.yaml new file mode 100644 index 0000000000000..1b37416d51ba6 --- /dev/null +++ b/docs/changelog/99286.yaml @@ -0,0 +1,5 @@ +pr: 99286 +summary: "ESQL: Log execution time consistently" +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java index ec2729e1b0fd0..0e0e9a4599780 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.rest.ChunkedRestResponseBody; @@ -26,7 +27,51 @@ import static org.elasticsearch.xpack.esql.formatter.TextFormat.CSV; import static org.elasticsearch.xpack.esql.formatter.TextFormat.URL_PARAM_DELIMITER; +/** + * Listens for a single {@link EsqlQueryResponse}, builds a corresponding {@link RestResponse} and sends it. + */ public class EsqlResponseListener extends RestResponseListener { + /** + * A simple, thread-safe stop watch for timing a single action. + * Allows to stop the time for building a response and to log it at a later point. + */ + private static class ThreadSafeStopWatch { + /** + * Start time of the watch + */ + private final long startTimeNS = System.nanoTime(); + + /** + * End time of the watch + */ + private long endTimeNS; + + /** + * Is the stop watch currently running? + */ + private boolean running = true; + + /** + * Starts the {@link ThreadSafeStopWatch} immediately after construction. + */ + ThreadSafeStopWatch() {} + + /** + * Stop the stop watch (or do nothing if it was already stopped) and return the elapsed time since starting. + * @return the elapsed time since starting the watch + */ + public TimeValue stop() { + synchronized (this) { + if (running) { + endTimeNS = System.nanoTime(); + running = false; + } + + return new TimeValue(endTimeNS - startTimeNS, TimeUnit.NANOSECONDS); + } + } + } + private static final Logger LOGGER = LogManager.getLogger(EsqlResponseListener.class); private static final String HEADER_NAME_TOOK_NANOS = "Took-nanos"; private final RestChannel channel; @@ -36,8 +81,15 @@ public class EsqlResponseListener extends RestResponseListener wrapWithLogging() { - // We need to measure the execution time after handling the response/failure for the measurement to be correct. return ActionListener.wrap(r -> { onResponse(r); - LOGGER.info("Successfully executed ESQL query in {}ms:\n{}", timeMillis(), esqlQuery); + // At this point, the StopWatch should already have been stopped, so we log a consistent time. + LOGGER.info("Successfully executed ESQL query in {}ms:\n{}", stopWatch.stop().getMillis(), esqlQuery); }, ex -> { + // In case of failure, stop the time manually before sending out the response. + long timeMillis = stopWatch.stop().getMillis(); onFailure(ex); - LOGGER.info("Failed executing ESQL query in {}ms:\n{}", timeMillis(), esqlQuery); + LOGGER.info("Failed executing ESQL query in {}ms:\n{}", timeMillis, esqlQuery); }); } - - private long timeMillis() { - long timeNanos = System.nanoTime() - startNanos; - return TimeUnit.NANOSECONDS.toMillis(timeNanos); - } } From 5822cc91c8be98a83149bd6fbf4cdd8035035b62 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 8 Sep 2023 17:44:26 +0100 Subject: [PATCH 45/69] Remove string-based Scheduler#schedule override (#99379) Completes the removal of the functionality deprecated by #99027. --- .../ScheduledExecutorServiceScheduler.java | 20 ++++++------------- .../elasticsearch/threadpool/Scheduler.java | 6 ------ .../elasticsearch/threadpool/ThreadPool.java | 10 ---------- .../action/bulk/BulkProcessorTests.java | 6 ------ 4 files changed, 6 insertions(+), 36 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/threadpool/ScheduledExecutorServiceScheduler.java b/server/src/main/java/org/elasticsearch/threadpool/ScheduledExecutorServiceScheduler.java index 6c0d8e30d4e84..cc7aad76cf6b7 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ScheduledExecutorServiceScheduler.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ScheduledExecutorServiceScheduler.java @@ -8,6 +8,7 @@ package org.elasticsearch.threadpool; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import java.util.concurrent.Executor; @@ -15,8 +16,8 @@ import java.util.concurrent.TimeUnit; /** - * A {@link Scheduler} which wraps a {@link ScheduledExecutorService}. It ignores the supplied {@code executor} or {@code executorName} and - * instead uses the inner executor to execute the delayed commands. + * A {@link Scheduler} which wraps a {@link ScheduledExecutorService}. It always runs the delayed command on the scheduler thread, so the + * provided {@link Executor} must always be {@link EsExecutors#DIRECT_EXECUTOR_SERVICE}. */ public final class ScheduledExecutorServiceScheduler implements Scheduler { private final ScheduledExecutorService executor; @@ -25,18 +26,9 @@ public ScheduledExecutorServiceScheduler(ScheduledExecutorService executor) { this.executor = executor; } - private ScheduledCancellable schedule(Runnable command, TimeValue delay) { - return Scheduler.wrapAsScheduledCancellable(executor.schedule(command, delay.millis(), TimeUnit.MILLISECONDS)); - } - @Override - public ScheduledCancellable schedule(Runnable command, TimeValue delay, Executor executor) { - return schedule(command, delay); - } - - @SuppressWarnings("removal") - @Override - public ScheduledCancellable schedule(Runnable command, TimeValue delay, String executorName) { - return schedule(command, delay); + public ScheduledCancellable schedule(Runnable command, TimeValue delay, Executor unused) { + assert unused == EsExecutors.DIRECT_EXECUTOR_SERVICE : "ScheduledExecutorServiceScheduler never forks, don't even try"; + return Scheduler.wrapAsScheduledCancellable(executor.schedule(command, delay.millis(), TimeUnit.MILLISECONDS)); } } diff --git a/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java b/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java index ad724aa190e18..438a32a2b8630 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java +++ b/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java @@ -95,12 +95,6 @@ static boolean awaitTermination( */ ScheduledCancellable schedule(Runnable command, TimeValue delay, Executor executor); - /** - * @deprecated Use {@link #schedule(Runnable, TimeValue, Executor)} instead. - */ - @Deprecated(forRemoval = true) - ScheduledCancellable schedule(Runnable command, TimeValue delay, String executorName); - /** * Schedules a periodic action that runs on scheduler thread. Do not run blocking calls on the scheduler thread. Subclasses may allow * to execute on a different executor, in which case blocking calls are allowed. diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 57ad689c3c5eb..bfcd8c8a396f5 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -453,16 +453,6 @@ public ExecutorService executor(String name) { return holder.executor(); } - /** - * @deprecated Use {@link #schedule(Runnable, TimeValue, Executor)} instead. - */ - @Override - @SuppressWarnings("removal") - @Deprecated(forRemoval = true) - public final ScheduledCancellable schedule(Runnable command, TimeValue delay, String executor) { - return schedule(command, delay, executor(executor)); - } - /** * Schedules a one-shot command to run after a given delay. The command is run in the context of the calling thread. * diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java index cd5652d25089e..09f79d1e48d0d 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java @@ -554,11 +554,5 @@ private static class UnusedScheduler implements Scheduler { public ScheduledCancellable schedule(Runnable command, TimeValue delay, Executor executor) { throw new AssertionError("should not be called"); } - - @SuppressWarnings("removal") - @Override - public ScheduledCancellable schedule(Runnable command, TimeValue delay, String executorName) { - throw new AssertionError("should not be called"); - } } } From df38bc8a84a7a47d98d16680f898596edb039122 Mon Sep 17 00:00:00 2001 From: Dianna Hohensee Date: Fri, 8 Sep 2023 13:21:53 -0400 Subject: [PATCH 46/69] Call TransportService#registerRequestHandler with Executor (#98998) Remove temporary TransportService#registerRequestHandler functions that take String instead of Executor and update the callers. This is part of a larger change to pass Executor into request handlers, rather than creating and stashing Strings that are only used to eventually fetch appropriate Executors anyway. Extensive testing changes were required to make TransportService and ThreadPool mocks capable of looking up Executors from Strings for TransportAction constructors. Added a new MockUtils class to modularize the additional test setup. Part of the changes for #97879 --- .../common/GrokProcessorGetActionTests.java | 7 +- .../TransportRankEvalActionTests.java | 4 +- .../reindex/UpdateByQueryWithScriptTests.java | 5 + ...rossClusterSearchUnavailableClusterIT.java | 54 ++- .../flush/TransportShardFlushAction.java | 2 +- .../TransportFieldCapabilitiesAction.java | 2 +- .../action/search/SearchTransportService.java | 25 +- .../TransportOpenPointInTimeAction.java | 3 +- .../support/HandledTransportAction.java | 9 +- .../broadcast/TransportBroadcastAction.java | 2 +- .../node/TransportBroadcastByNodeAction.java | 2 +- .../TransportBroadcastUnpromotableAction.java | 7 +- .../support/nodes/TransportNodesAction.java | 2 +- .../TransportReplicationAction.java | 12 +- ...ransportInstanceSingleOperationAction.java | 4 +- .../shard/TransportSingleShardAction.java | 10 +- .../support/tasks/TransportTasksAction.java | 7 +- .../action/shard/ShardStateAction.java | 5 +- .../cluster/coordination/Coordinator.java | 2 +- .../coordination/FollowersChecker.java | 2 +- .../cluster/coordination/JoinHelper.java | 7 +- .../coordination/JoinValidationService.java | 2 +- .../cluster/coordination/LeaderChecker.java | 2 +- .../PublicationTransportHandler.java | 2 +- .../StatefulPreVoteCollector.java | 2 +- .../elasticsearch/discovery/PeerFinder.java | 2 +- .../gateway/LocalAllocateDangledIndices.java | 3 +- .../recovery/PeerRecoverySourceService.java | 4 +- .../recovery/PeerRecoveryTargetService.java | 16 +- .../indices/store/IndicesStore.java | 3 +- .../VerifyNodeRepositoryAction.java | 2 +- .../tasks/TaskCancellationService.java | 5 +- .../transport/RemoteClusterService.java | 3 +- .../transport/TransportActionProxy.java | 3 +- .../transport/TransportService.java | 40 -- ...nsportDeleteDesiredBalanceActionTests.java | 16 +- ...TransportGetDesiredBalanceActionTests.java | 39 +- .../ClusterFormationInfoActionTests.java | 6 + .../MasterHistoryActionTests.java | 6 + ...ransportUpdateDesiredNodesActionTests.java | 20 +- .../ReservedRepositoryActionTests.java | 7 +- .../settings/ClusterGetSettingsTests.java | 7 +- .../ClusterUpdateSettingsRequestTests.java | 7 +- .../TransportCreateIndexActionTests.java | 8 +- .../TransportRolloverActionTests.java | 6 + .../TransportUpdateSettingsActionTests.java | 8 +- ...vedComposableIndexTemplateActionTests.java | 25 +- ...ActionIndicesThatCannotBeCreatedTests.java | 10 +- .../bulk/TransportBulkActionIngestTests.java | 5 + ...StateAwareHandledTransportActionTests.java | 6 +- .../replication/PostWriteRefreshTests.java | 4 +- .../TransportWriteActionTests.java | 2 +- .../PeerRecoverySourceServiceTests.java | 4 +- .../tasks/BanFailureLoggingTests.java | 2 +- .../RemoteClusterConnectionTests.java | 78 ++-- .../SniffConnectionStrategyTests.java | 5 +- .../transport/TransportActionProxyTests.java | 99 +++-- ...ortServiceDeserializationFailureTests.java | 3 +- .../TransportServiceLifecycleTests.java | 2 +- .../org/elasticsearch/test/MockUtils.java | 71 +++ .../AbstractSimpleTransportTestCase.java | 410 +++++++++++------- .../DisruptableMockTransportTests.java | 7 +- .../AnalyticsInfoTransportActionTests.java | 17 +- .../TransportAnalyticsStatsActionTests.java | 6 + ...ortDeleteAutoscalingPolicyActionTests.java | 13 +- ...nsportGetAutoscalingPolicyActionTests.java | 13 +- ...nsportPutAutoscalingPolicyActionTests.java | 13 +- .../ccr/CCRInfoTransportActionTests.java | 18 +- .../ClearCcrRestoreSessionActionTests.java | 3 + .../GetCcrRestoreFileChunkActionTests.java | 3 + .../action/TransportTermsEnumAction.java | 2 +- .../action/TransportXPackInfoActionTests.java | 4 +- ...nsportNodeDeprecationCheckActionTests.java | 7 + .../InternalExecutePolicyActionTests.java | 4 +- .../eql/EqlInfoTransportActionTests.java | 13 +- .../compute/operator/DriverTaskRunner.java | 4 +- .../operator/exchange/ExchangeService.java | 4 +- .../esql/enrich/EnrichLookupService.java | 7 +- .../xpack/esql/plugin/ComputeService.java | 9 +- .../esql/EsqlInfoTransportActionTests.java | 12 +- .../graph/GraphInfoTransportActionTests.java | 12 +- ...portPutSamlServiceProviderActionTests.java | 5 +- ...ndexLifecycleInfoTransportActionTests.java | 23 +- .../TransportDeleteLifecycleActionTests.java | 7 +- .../TransportPutLifecycleActionTests.java | 7 +- .../action/TransportStopILMActionTests.java | 7 +- .../TransportDeletePipelineActionTests.java | 4 +- .../TransportGetPipelineActionTests.java | 14 +- ...chineLearningInfoTransportActionTests.java | 14 +- .../action/TransportCloseJobActionTests.java | 7 +- ...TransportDeleteExpiredDataActionTests.java | 5 + ...nsportFinalizeJobExecutionActionTests.java | 6 +- .../MonitoringInfoTransportActionTests.java | 19 +- .../TransportMonitoringBulkActionTests.java | 5 + .../action/TransportRollupSearchAction.java | 4 +- .../RollupInfoTransportActionTests.java | 17 +- ...CrossClusterAccessHeadersForCcsRestIT.java | 60 +-- .../SecurityInfoTransportActionTests.java | 23 +- ...rtCreateCrossClusterApiKeyActionTests.java | 9 +- .../TransportGrantApiKeyActionTests.java | 8 +- ...rtUpdateCrossClusterApiKeyActionTests.java | 7 +- ...nsportProfileHasPrivilegesActionTests.java | 5 + .../TransportClearRealmCacheActionTests.java | 8 +- ...tCreateServiceAccountTokenActionTests.java | 5 +- ...tDeleteServiceAccountTokenActionTests.java | 4 +- ...TransportGetServiceAccountActionTests.java | 4 +- ...tServiceAccountCredentialsActionTests.java | 5 +- .../TransportCreateTokenActionTests.java | 16 +- .../TransportInvalidateTokenActionTests.java | 10 +- .../TransportHasPrivilegesActionTests.java | 9 +- ...ty4ServerTransportAuthenticationTests.java | 3 +- ...ransportDeleteShutdownNodeActionTests.java | 4 +- .../TransportPutShutdownNodeActionTests.java | 4 +- ...vedSnapshotLifecycleStateServiceTests.java | 13 +- .../action/TransportStopSLMActionTests.java | 8 +- .../SpatialInfoTransportActionTests.java | 17 +- .../SpatialStatsTransportActionTests.java | 3 +- .../sql/SqlInfoTransportActionTests.java | 14 +- .../TransformInfoTransportActionTests.java | 7 +- .../WatcherInfoTransportActionTests.java | 14 +- .../actions/TransportAckWatchActionTests.java | 3 +- .../actions/TransportPutWatchActionTests.java | 3 +- .../TransportWatcherStatsActionTests.java | 3 +- 123 files changed, 1074 insertions(+), 628 deletions(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/test/MockUtils.java diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java index c2110a14db7a6..09801eeda6437 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.grok.PatternBank; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -60,8 +61,9 @@ public void testResponseSerialization() throws Exception { public void testResponseSorting() { List sortedKeys = new ArrayList<>(LEGACY_TEST_PATTERNS.bank().keySet()); Collections.sort(sortedKeys); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GrokProcessorGetAction.TransportAction transportAction = new GrokProcessorGetAction.TransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), LEGACY_TEST_PATTERNS, ECS_TEST_PATTERNS @@ -109,8 +111,9 @@ public void onFailure(Exception e) { public void testEcsCompatibilityMode() { List sortedKeys = new ArrayList<>(ECS_TEST_PATTERNS.bank().keySet()); Collections.sort(sortedKeys); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GrokProcessorGetAction.TransportAction transportAction = new GrokProcessorGetAction.TransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), LEGACY_TEST_PATTERNS, ECS_TEST_PATTERNS diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java index 1b12f45ac8b51..3838916c15c64 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -74,10 +75,11 @@ public void multiSearch(MultiSearchRequest request, ActionListener { channel.sendResponse(new SearchShardsResponse(List.of(), List.of(), Collections.emptyMap())); } ); - newService.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { - InternalSearchResponse response = new InternalSearchResponse( - new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - null, - false, - null, - 1 - ); - SearchResponse searchResponse = new SearchResponse( - response, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); - channel.sendResponse(searchResponse); - }); + newService.registerRequestHandler( + SearchAction.NAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SearchRequest::new, + (request, channel, task) -> { + InternalSearchResponse response = new InternalSearchResponse( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), + InternalAggregations.EMPTY, + null, + null, + false, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + response, + null, + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + channel.sendResponse(searchResponse); + } + ); newService.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java index 0bd4458056c20..fbcfe415039ff 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java @@ -62,7 +62,7 @@ public TransportShardFlushAction( ); transportService.registerRequestHandler( PRE_SYNCED_FLUSH_ACTION_NAME, - ThreadPool.Names.FLUSH, + threadPool.executor(ThreadPool.Names.FLUSH), PreShardSyncedFlushRequest::new, new PreSyncedFlushTransportHandler(indicesService) ); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 79b9fa5099467..1dde340e0f527 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -92,7 +92,7 @@ public TransportFieldCapabilitiesAction( this.indicesService = indicesService; transportService.registerRequestHandler( ACTION_NODE_NAME, - ThreadPool.Names.SEARCH_COORDINATION, + this.searchCoordinationExecutor, FieldCapabilitiesNodeRequest::new, new NodeTransportHandler() ); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java index 9714fc7574418..d02958567a873 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.search.CanMatchShardResponse; import org.elasticsearch.search.SearchPhaseResult; @@ -450,7 +451,7 @@ public void writeTo(StreamOutput out) throws IOException { public static void registerRequestHandler(TransportService transportService, SearchService searchService) { transportService.registerRequestHandler( FREE_CONTEXT_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ScrollFreeContextRequest::new, (request, channel, task) -> { boolean freed = searchService.freeReaderContext(request.id()); @@ -460,7 +461,7 @@ public static void registerRequestHandler(TransportService transportService, Sea TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_SCROLL_ACTION_NAME, false, SearchFreeContextResponse::new); transportService.registerRequestHandler( FREE_CONTEXT_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, SearchFreeContextRequest::new, (request, channel, task) -> { boolean freed = searchService.freeReaderContext(request.id()); @@ -470,7 +471,7 @@ public static void registerRequestHandler(TransportService transportService, Sea TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, false, SearchFreeContextResponse::new); transportService.registerRequestHandler( CLEAR_SCROLL_CONTEXTS_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, TransportRequest.Empty::new, (request, channel, task) -> { searchService.freeAllScrollContexts(); @@ -486,7 +487,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( DFS_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardSearchRequest::new, (request, channel, task) -> searchService.executeDfsPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)) ); @@ -495,7 +496,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardSearchRequest::new, (request, channel, task) -> searchService.executeQueryPhase( request, @@ -512,7 +513,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_ID_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, QuerySearchRequest::new, (request, channel, task) -> { searchService.executeQueryPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -522,7 +523,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, InternalScrollSearchRequest::new, (request, channel, task) -> { searchService.executeQueryPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -532,7 +533,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_FETCH_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, InternalScrollSearchRequest::new, (request, channel, task) -> { searchService.executeFetchPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -542,7 +543,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( FETCH_ID_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardFetchRequest::new, (request, channel, task) -> { searchService.executeFetchPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -552,7 +553,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( FETCH_ID_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, true, true, ShardFetchSearchRequest::new, @@ -565,7 +566,7 @@ public static void registerRequestHandler(TransportService transportService, Sea // this is cheap, it does not fetch during the rewrite phase, so we can let it quickly execute on a networking thread transportService.registerRequestHandler( QUERY_CAN_MATCH_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardSearchRequest::new, (request, channel, task) -> { searchService.canMatch(request, new ChannelActionListener<>(channel)); @@ -575,7 +576,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_CAN_MATCH_NODE_NAME, - ThreadPool.Names.SEARCH_COORDINATION, + transportService.getThreadPool().executor(ThreadPool.Names.SEARCH_COORDINATION), CanMatchNodeRequest::new, (request, channel, task) -> { searchService.canMatch(request, new ChannelActionListener<>(channel)); diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java index 9c78e5ad62aea..eb854be99562e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchPhaseResult; @@ -68,7 +69,7 @@ public TransportOpenPointInTimeAction( this.searchTransportService = searchTransportService; transportService.registerRequestHandler( OPEN_SHARD_READER_CONTEXT_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardOpenReaderRequest::new, new ShardOpenReaderRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java index 24668d7130d52..a0e36a538913d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java @@ -61,7 +61,14 @@ protected HandledTransportAction( String executor ) { super(actionName, actionFilters, transportService.getTaskManager()); - transportService.registerRequestHandler(actionName, executor, false, canTripCircuitBreaker, requestReader, new TransportHandler()); + transportService.registerRequestHandler( + actionName, + transportService.getThreadPool().executor(executor), + false, + canTripCircuitBreaker, + requestReader, + new TransportHandler() + ); } class TransportHandler implements TransportRequestHandler { diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java index 570514355da56..ddd2cc43005e2 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java @@ -75,7 +75,7 @@ protected TransportBroadcastAction( transportService.registerRequestHandler( transportShardAction, - executor, + this.executor, shardRequestReader, (request, channel, task) -> ActionListener.completeWith( new ChannelActionListener<>(channel), diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index 15c28bdf33c87..4f78d68daa8f2 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -119,7 +119,7 @@ public TransportBroadcastByNodeAction( transportService.registerRequestHandler( transportNodeBroadcastAction, - executor, + this.executor, false, canTripCircuitBreaker, NodeRequest::new, diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java index d5c7fc763fba6..579ed101b56ae 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java @@ -60,7 +60,12 @@ protected TransportBroadcastUnpromotableAction( this.transportUnpromotableAction = actionName + "[u]"; this.executor = transportService.getThreadPool().executor(executor); - transportService.registerRequestHandler(transportUnpromotableAction, executor, requestReader, new UnpromotableTransportHandler()); + transportService.registerRequestHandler( + transportUnpromotableAction, + this.executor, + requestReader, + new UnpromotableTransportHandler() + ); } protected abstract void unpromotableShardOperation(Task task, Request request, ActionListener listener); diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java index 1efe70bcb5adc..b45c448ecc52c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java @@ -82,7 +82,7 @@ protected TransportNodesAction( this.transportService = Objects.requireNonNull(transportService); this.finalExecutor = threadPool.executor(executor); this.transportNodeAction = actionName + "[n]"; - transportService.registerRequestHandler(transportNodeAction, executor, nodeRequest, new NodeTransportHandler()); + transportService.registerRequestHandler(transportNodeAction, finalExecutor, nodeRequest, new NodeTransportHandler()); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 5651af11719cf..d1e2b6dd98faa 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; @@ -187,11 +188,16 @@ protected TransportReplicationAction( this.retryTimeout = REPLICATION_RETRY_TIMEOUT.get(settings); this.forceExecutionOnPrimary = forceExecutionOnPrimary; - transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, requestReader, this::handleOperationRequest); + transportService.registerRequestHandler( + actionName, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + requestReader, + this::handleOperationRequest + ); transportService.registerRequestHandler( transportPrimaryAction, - executor, + threadPool.executor(executor), forceExecutionOnPrimary, true, in -> new ConcreteShardRequest<>(requestReader, in), @@ -201,7 +207,7 @@ protected TransportReplicationAction( // we must never reject on because of thread pool capacity on replicas transportService.registerRequestHandler( transportReplicaAction, - executor, + threadPool.executor(executor), true, true, in -> new ConcreteReplicaRequest<>(replicaRequestReader, in), diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index 63ff4cef5a924..c526a2679306d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; @@ -33,7 +34,6 @@ import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.threadpool.ThreadPool.Names; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; @@ -72,7 +72,7 @@ protected TransportInstanceSingleOperationAction( this.transportService = transportService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.shardActionName = actionName + "[s]"; - transportService.registerRequestHandler(shardActionName, Names.SAME, request, new ShardTransportHandler()); + transportService.registerRequestHandler(shardActionName, EsExecutors.DIRECT_EXECUTOR_SERVICE, request, new ShardTransportHandler()); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java index 001a99ecd505d..9d980fabc6714 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.LoggerMessageFormat; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; @@ -79,9 +80,14 @@ protected TransportSingleShardAction( this.executor = executor; if (isSubAction() == false) { - transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, request, new TransportHandler()); + transportService.registerRequestHandler(actionName, EsExecutors.DIRECT_EXECUTOR_SERVICE, request, new TransportHandler()); } - transportService.registerRequestHandler(transportShardAction, ThreadPool.Names.SAME, request, new ShardTransportHandler()); + transportService.registerRequestHandler( + transportShardAction, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + request, + new ShardTransportHandler() + ); } /** diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index b509008af2b4f..fb29e8a838db6 100644 --- a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -79,7 +79,12 @@ protected TransportTasksAction( this.responsesReader = responsesReader; this.responseReader = responseReader; - transportService.registerRequestHandler(transportNodeAction, nodeExecutor, NodeTaskRequest::new, new NodeTransportHandler()); + transportService.registerRequestHandler( + transportNodeAction, + transportService.getThreadPool().executor(nodeExecutor), + NodeTaskRequest::new, + new NodeTransportHandler() + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index 97bf6cc01c4e5..f9aa2c6c234be 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; @@ -98,13 +99,13 @@ public ShardStateAction( transportService.registerRequestHandler( SHARD_STARTED_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, StartedShardEntry::new, new ShardStartedTransportHandler(clusterService, new ShardStartedClusterStateTaskExecutor(allocationService, rerouteService)) ); transportService.registerRequestHandler( SHARD_FAILED_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, FailedShardEntry::new, new ShardFailedTransportHandler(clusterService, new ShardFailedClusterStateTaskExecutor(allocationService, rerouteService)) ); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index dc52791a5d5e2..7ccea8e99918b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -271,7 +271,7 @@ public Coordinator( ); transportService.registerRequestHandler( COMMIT_STATE_ACTION_NAME, - Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, ApplyCommitRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java index ad2faaccf0e96..99ab8650a52d8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java @@ -124,7 +124,7 @@ public FollowersChecker( updateFastResponseState(0, Mode.CANDIDATE); transportService.registerRequestHandler( FOLLOWER_CHECK_ACTION_NAME, - Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, FollowerCheckRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java index ce2754fa3854c..247034c88ed62 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; @@ -114,7 +115,7 @@ public class JoinHelper { transportService.registerRequestHandler( JOIN_ACTION_NAME, - Names.CLUSTER_COORDINATION, + transportService.getThreadPool().executor(Names.CLUSTER_COORDINATION), false, false, JoinRequest::new, @@ -126,7 +127,7 @@ public class JoinHelper { transportService.registerRequestHandler( START_JOIN_ACTION_NAME, - Names.CLUSTER_COORDINATION, + transportService.getThreadPool().executor(Names.CLUSTER_COORDINATION), false, false, StartJoinRequest::new, @@ -139,7 +140,7 @@ public class JoinHelper { transportService.registerRequestHandler( JOIN_PING_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, TransportRequest.Empty::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java index 248137f03fcaa..d9911ad12df84 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java @@ -121,7 +121,7 @@ public JoinValidationService( final var dataPaths = Environment.PATH_DATA_SETTING.get(settings); transportService.registerRequestHandler( JoinValidationService.JOIN_VALIDATE_ACTION_NAME, - ThreadPool.Names.CLUSTER_COORDINATION, + this.responseExecutor, ValidateJoinRequest::new, (request, channel, task) -> { final var remoteState = request.getOrReadState(); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java index 8a20e8e56d751..9fcae5bcf67f8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java @@ -113,7 +113,7 @@ public class LeaderChecker { transportService.registerRequestHandler( LEADER_CHECK_ACTION_NAME, - Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, LeaderCheckRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java index 45079b2bccd60..781a05d535b16 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java @@ -106,7 +106,7 @@ public PublicationTransportHandler( transportService.registerRequestHandler( PUBLISH_STATE_ACTION_NAME, - ThreadPool.Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, BytesTransportRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java index 28ab04e5a7ccc..7bc3514206ffe 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java @@ -64,7 +64,7 @@ public StatefulPreVoteCollector( transportService.registerRequestHandler( REQUEST_PRE_VOTE_ACTION_NAME, - Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, PreVoteRequest::new, diff --git a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java index 046f0a1c64bb5..46e8f5210e447 100644 --- a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java +++ b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java @@ -110,7 +110,7 @@ public PeerFinder( transportService.registerRequestHandler( REQUEST_PEERS_ACTION_NAME, - Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, PeersRequest::new, diff --git a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java index d0579342f5bd0..0262e37dd74a9 100644 --- a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java +++ b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java @@ -35,7 +35,6 @@ import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestHandler; @@ -73,7 +72,7 @@ public LocalAllocateDangledIndices( this.indexMetadataVerifier = indexMetadataVerifier; transportService.registerRequestHandler( ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, AllocateDangledRequest::new, new AllocateDangledRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java index 21b8ca31f01a7..eac119f920f6a 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java @@ -79,7 +79,7 @@ public PeerRecoverySourceService( // node. Upon receiving START_RECOVERY, the source node will initiate the peer recovery. transportService.registerRequestHandler( Actions.START_RECOVERY, - ThreadPool.Names.GENERIC, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), StartRecoveryRequest::new, (request, channel, task) -> recover(request, task, new ChannelActionListener<>(channel)) ); @@ -89,7 +89,7 @@ public PeerRecoverySourceService( // action will fail and the target node will send a new START_RECOVERY request. transportService.registerRequestHandler( Actions.REESTABLISH_RECOVERY, - ThreadPool.Names.GENERIC, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), ReestablishRecoveryRequest::new, (request, channel, task) -> reestablish(request, new ChannelActionListener<>(channel)) ); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java index 5f06519f84c0c..b3b23ac14d158 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java @@ -122,7 +122,7 @@ public PeerRecoveryTargetService( transportService.registerRequestHandler( Actions.FILES_INFO, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryFilesInfoRequest::new, new RecoveryRequestHandler<>() { @Override @@ -140,7 +140,7 @@ protected void handleRequest(RecoveryFilesInfoRequest request, RecoveryTarget ta ); transportService.registerRequestHandler( Actions.RESTORE_FILE_FROM_SNAPSHOT, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoverySnapshotFileRequest::new, new RecoveryRequestHandler<>() { @Override @@ -151,13 +151,13 @@ protected void handleRequest(RecoverySnapshotFileRequest request, RecoveryTarget ); transportService.registerRequestHandler( Actions.FILE_CHUNK, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryFileChunkRequest::new, new FileChunkTransportRequestHandler() ); transportService.registerRequestHandler( Actions.CLEAN_FILES, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryCleanFilesRequest::new, new RecoveryRequestHandler<>() { @Override @@ -179,7 +179,7 @@ protected void handleRequest(RecoveryCleanFilesRequest request, RecoveryTarget t ); transportService.registerRequestHandler( Actions.PREPARE_TRANSLOG, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryPrepareForTranslogOperationsRequest::new, new RecoveryRequestHandler<>() { @Override @@ -194,13 +194,13 @@ protected void handleRequest( ); transportService.registerRequestHandler( Actions.TRANSLOG_OPS, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryTranslogOperationsRequest::new, new TranslogOperationsRequestHandler() ); transportService.registerRequestHandler( Actions.FINALIZE, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryFinalizeRecoveryRequest::new, new RecoveryRequestHandler<>() { @Override @@ -215,7 +215,7 @@ protected void handleRequest( ); transportService.registerRequestHandler( Actions.HANDOFF_PRIMARY_CONTEXT, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryHandoffPrimaryContextRequest::new, new HandoffPrimaryContextRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index e12a38e383397..f949dd59e8968 100644 --- a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -102,7 +103,7 @@ public IndicesStore( this.threadPool = threadPool; transportService.registerRequestHandler( ACTION_SHARD_EXISTS, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardActiveRequest::new, new ShardActiveRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java index ad6cbbaad1cdc..d940415c38916 100644 --- a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java @@ -56,7 +56,7 @@ public VerifyNodeRepositoryAction( this.repositoriesService = repositoriesService; transportService.registerRequestHandler( ACTION_NAME, - ThreadPool.Names.SNAPSHOT, + transportService.getThreadPool().executor(ThreadPool.Names.SNAPSHOT), VerifyNodeRepositoryRequest::new, new VerifyNodeRepositoryRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java index 39a76f3508daa..6ab95072727c0 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.NodeDisconnectedException; @@ -60,13 +61,13 @@ public TaskCancellationService(TransportService transportService) { this.deduplicator = new ResultDeduplicator<>(transportService.getThreadPool().getThreadContext()); transportService.registerRequestHandler( BAN_PARENT_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, BanParentTaskRequest::new, new BanParentRequestHandler() ); transportService.registerRequestHandler( CANCEL_CHILD_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, CancelChildRequest::new, new CancelChildRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index 9542d4b366ded..c38f4b26c665f 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.node.ReportingService; @@ -539,7 +540,7 @@ Collection getConnections() { static void registerRemoteClusterHandshakeRequestHandler(TransportService transportService) { transportService.registerRequestHandler( REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, TransportService.HandshakeRequest::new, diff --git a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java index ed3432f66087b..aa28f8a76b58e 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -188,7 +189,7 @@ public static void registerProxyActionWithDynamicResponseType( RequestHandlerRegistry requestHandler = service.getRequestHandler(action); service.registerRequestHandler( getProxyAction(action), - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, true, false, in -> cancellable diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 83fc21396e0f6..5369b9a9eec13 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -1118,46 +1118,6 @@ public static boolean isValidActionName(String actionName) { return false; } - /** - * Temporary passthrough function that continues to take a String rather than Executor type. - * - * @param action - * @param executor - * @param requestReader - * @param handler - * @param - */ - public void registerRequestHandler( - String action, - String executor, - Writeable.Reader requestReader, - TransportRequestHandler handler - ) { - registerRequestHandler(action, threadPool.executor(executor), requestReader, handler); - } - - /** - * Temporary passthrough function that continues to take a String rather than Executor type. - * - * @param action - * @param executor - * @param forceExecution - * @param canTripCircuitBreaker - * @param requestReader - * @param handler - * @param - */ - public void registerRequestHandler( - String action, - String executor, - boolean forceExecution, - boolean canTripCircuitBreaker, - Writeable.Reader requestReader, - TransportRequestHandler handler - ) { - registerRequestHandler(action, threadPool.executor(executor), forceExecution, canTripCircuitBreaker, requestReader, handler); - } - /** * Registers a new request handler * diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java index f5e812e7d0983..a2bfa68fdf25f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.gateway.TestGatewayAllocator; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -56,17 +57,20 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportDeleteDesiredBalanceActionTests extends ESAllocationTestCase { public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception { - var listener = new PlainActionFuture(); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + new TransportDeleteDesiredBalanceAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(AllocationService.class), @@ -131,8 +135,12 @@ public DesiredBalance compute( var listener = new PlainActionFuture(); + // TODO: temporary, remove in #97879 + TransportService transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + var action = new TransportDeleteDesiredBalanceAction( - mock(TransportService.class), + transportService, clusterService, threadPool, mock(ActionFilters.class), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java index 804dbf66ac0d0..80f0b435645e6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; @@ -45,6 +46,7 @@ import org.elasticsearch.test.AbstractChunkedSerializingTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.junit.Before; import java.util.HashMap; import java.util.List; @@ -59,6 +61,7 @@ import static org.elasticsearch.cluster.ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -66,16 +69,27 @@ public class TransportGetDesiredBalanceActionTests extends ESAllocationTestCase private final DesiredBalanceShardsAllocator desiredBalanceShardsAllocator = mock(DesiredBalanceShardsAllocator.class); private final ClusterInfoService clusterInfoService = mock(ClusterInfoService.class); - private final TransportGetDesiredBalanceAction transportGetDesiredBalanceAction = new TransportGetDesiredBalanceAction( - mock(TransportService.class), - mock(ClusterService.class), - mock(ThreadPool.class), - mock(ActionFilters.class), - mock(IndexNameExpressionResolver.class), - desiredBalanceShardsAllocator, - clusterInfoService, - TEST_WRITE_LOAD_FORECASTER - ); + private TransportService transportService = mock(TransportService.class); + private ThreadPool threadPool = mock(ThreadPool.class); + private TransportGetDesiredBalanceAction transportGetDesiredBalanceAction; + + @Before + public void initialize() { + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + + transportGetDesiredBalanceAction = new TransportGetDesiredBalanceAction( + transportService, + mock(ClusterService.class), + threadPool, + mock(ActionFilters.class), + mock(IndexNameExpressionResolver.class), + desiredBalanceShardsAllocator, + clusterInfoService, + TEST_WRITE_LOAD_FORECASTER + ); + } private static DesiredBalanceResponse execute(TransportGetDesiredBalanceAction action, ClusterState clusterState) throws Exception { return PlainActionFuture.get( @@ -96,11 +110,10 @@ private DesiredBalanceResponse executeAction(ClusterState clusterState) throws E public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception { var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataWithConfiguredAllocator(BALANCED_ALLOCATOR)).build(); - final var action = new TransportGetDesiredBalanceAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(ShardsAllocator.class), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java index e560959dfacf8..f190c422e165c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.monitor.StatusInfo; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Map; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -162,6 +164,10 @@ public void testTransportDoExecute() { Coordinator coordinator = mock(Coordinator.class); ClusterFormationFailureHelper.ClusterFormationState clusterFormationState = getClusterFormationState(); when(coordinator.getClusterFormationState()).thenReturn(clusterFormationState); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); ClusterFormationInfoAction.TransportAction action = new ClusterFormationInfoAction.TransportAction( transportService, actionFilters, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java index 3674a711d8968..f362327a3d8e4 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; import org.elasticsearch.threadpool.ThreadPool; @@ -24,6 +25,7 @@ import java.util.ArrayList; import java.util.List; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -83,6 +85,10 @@ public void testTransportDoExecute() { when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis()); MasterHistory masterHistory = new MasterHistory(threadPool, clusterService); when(masterHistoryService.getLocalMasterHistory()).thenReturn(masterHistory); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); MasterHistoryAction.TransportAction action = new MasterHistoryAction.TransportAction( transportService, actionFilters, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java index e4e8293810f56..508cb9b304c59 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -54,10 +55,12 @@ public void validate(List desiredNodes) {} }; public void testWriteBlocks() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_OP_SETTINGS_VALIDATOR, @@ -79,10 +82,12 @@ public void testWriteBlocks() { } public void testNoBlocks() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_OP_SETTINGS_VALIDATOR, @@ -103,10 +108,13 @@ public void validate(List desiredNodes) { } }; ClusterService clusterService = mock(ClusterService.class); + + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), validator, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java index 3b3a996e50cd3..1b3e0ae6fe7bf 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentParser; @@ -132,14 +133,16 @@ public Repository create(RepositoryMetadata metadata) { } }; + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); RepositoriesService repositoriesService = spy( new RepositoriesService( Settings.EMPTY, mock(ClusterService.class), - mock(TransportService.class), + transportService, Map.of(), Map.of("fs", fsFactory), - mock(ThreadPool.class), + threadPool, null ) ); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java index 9e757894bcdda..00b500254883c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -47,10 +48,12 @@ public void testRequestConstruction() { public void testTransportFilters() throws Exception { final SettingsFilter filter = new SettingsFilter(List.of("persistent.foo.filtered", "transient.foo.filtered")); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportClusterGetSettingsAction action = new TransportClusterGetSettingsAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, filter, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java index ebe77f4d3ed4c..6373b94ffb94a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -85,10 +86,12 @@ private static ClusterUpdateSettingsRequest createTestItem() { public void testOperatorHandler() throws IOException { ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportClusterUpdateSettingsAction action = new TransportClusterUpdateSettingsAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), clusterSettings diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java index 671bf13fb0e77..d68641d04dd74 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -90,10 +91,13 @@ public void setUp() throws Exception { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, SYSTEM_INDICES); this.metadataCreateIndexService = mock(MetadataCreateIndexService.class); + + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); this.action = new TransportCreateIndexAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, metadataCreateIndexService, mock(ActionFilters.class), indexNameExpressionResolver, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 12308036689c6..bb1cf85013498 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.cache.query.QueryCacheStats; @@ -78,6 +79,7 @@ import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -354,6 +356,10 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr EmptySystemIndices.INSTANCE, WriteLoadForecaster.DEFAULT ); + + // TODO: temporary, remove in #97879 + when(mockTransportService.getThreadPool()).thenReturn(mockThreadPool); + when(mockThreadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); final TransportRolloverAction transportRolloverAction = new TransportRolloverAction( mockTransportService, mockClusterService, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java index 0e1581606f803..807462b6052c3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -68,10 +69,13 @@ public void setUp() throws Exception { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, SYSTEM_INDICES); MetadataUpdateSettingsService metadataUpdateSettingsService = mock(MetadataUpdateSettingsService.class); + + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); this.action = new TransportUpdateSettingsAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, metadataUpdateSettingsService, mock(ActionFilters.class), indexNameExpressionResolver, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java index 553dd2368e5bc..28476a0d8b839 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -673,10 +674,12 @@ public void testAddRemoveIndexTemplatesWithOverlap() throws Exception { } public void testHandlerCorrectness() { + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putIndexAction = new TransportPutComposableIndexTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null @@ -687,9 +690,9 @@ public void testHandlerCorrectness() { containsInAnyOrder(reservedComposableIndexName("aaa")) ); var delIndexAction = new TransportDeleteComposableIndexTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null @@ -701,9 +704,9 @@ public void testHandlerCorrectness() { ); var putComponentAction = new TransportPutComponentTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null, @@ -716,9 +719,9 @@ public void testHandlerCorrectness() { ); var delComponentAction = new TransportDeleteComponentTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null @@ -921,10 +924,12 @@ public void testTemplatesWithReservedPrefix() throws Exception { PutComposableIndexTemplateAction.Request pr = new PutComposableIndexTemplateAction.Request(conflictingTemplateName); + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putTemplateAction = new TransportPutComposableIndexTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index ff96f4a00b883..1b0c24664be31 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; @@ -32,6 +31,7 @@ import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -42,7 +42,6 @@ import java.util.function.Function; import static java.util.Collections.emptySet; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -107,9 +106,6 @@ private void indicesThatCannotBeCreatedTestCase( when(clusterService.localNode()).thenReturn(localNode); when(localNode.isIngestNode()).thenReturn(randomBoolean()); - final ThreadPool threadPool = mock(ThreadPool.class); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver( new ThreadContext(Settings.EMPTY), EmptySystemIndices.INSTANCE @@ -120,9 +116,11 @@ public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) } }; + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportBulkAction action = new TransportBulkAction( threadPool, - mock(TransportService.class), + transportService, clusterService, null, null, diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index fd84a1e5fe816..85a74174b094f 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -183,6 +183,11 @@ public void setupAction() { MockitoAnnotations.openMocks(this); // setup services that will be called by action transportService = mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + clusterService = mock(ClusterService.class); localIngest = true; // setup nodes for local and remote diff --git a/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java b/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java index 93c4ea98adcd3..c2ff59f0ccbe0 100644 --- a/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -47,7 +48,8 @@ public void testRejectImmutableConflictClusterStateUpdate() { ClusterService clusterService = mock(ClusterService.class); doReturn(clusterState).when(clusterService).state(); - Action handler = new Action("internal:testAction", clusterService, mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + Action handler = new Action("internal:testAction", clusterService, transportService, mock(ActionFilters.class)); // nothing should happen here, since the request doesn't touch any of the immutable state keys var future = new PlainActionFuture(); @@ -61,7 +63,7 @@ public void testRejectImmutableConflictClusterStateUpdate() { FakeReservedStateAwareAction action = new FakeReservedStateAwareAction( "internal:testClusterSettings", clusterService, - mock(TransportService.class), + transportService, mock(ActionFilters.class), null ); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java index e45fc51cf6bdd..d14429647c7d3 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.engine.DocIdSeqNoAndSource; @@ -31,7 +32,6 @@ import org.elasticsearch.index.shard.ReplicationGroup; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -67,7 +67,7 @@ public void setUp() throws Exception { transportService.acceptIncomingRequests(); transportService.registerRequestHandler( TransportUnpromotableShardRefreshAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, UnpromotableShardRefreshRequest::new, (request, channel, task) -> { unpromotableRefreshRequestReceived.set(true); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java index 8e95277bdc84f..7726feaa30868 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -424,7 +424,7 @@ protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentF ), TransportWriteActionTests.this.clusterService, null, - null, + TransportWriteActionTests.threadPool, null, new ActionFilters(new HashSet<>()), TestRequest::new, diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java index 66503bf4cea79..a18e7e8ce46f9 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.NodeRoles; import org.elasticsearch.transport.TransportService; @@ -38,8 +39,9 @@ public void testDuplicateRecoveries() throws IOException { final ClusterService clusterService = mock(ClusterService.class); when(clusterService.getSettings()).thenReturn(NodeRoles.dataNode()); when(indicesService.clusterService()).thenReturn(clusterService); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService( - mock(TransportService.class), + transportService, indicesService, new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), mock(RecoveryPlannerService.class) diff --git a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java index 4ffd1e3ed94eb..56e72c25802e3 100644 --- a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java +++ b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java @@ -132,7 +132,7 @@ private void runTest( childTransportService.getTaskManager().setTaskCancellationService(new TaskCancellationService(childTransportService)); childTransportService.registerRequestHandler( "internal:testAction[c]", - ThreadPool.Names.MANAGEMENT, // busy-wait for cancellation but not on a transport thread + threadPool.executor(ThreadPool.Names.MANAGEMENT), // busy-wait for cancellation but not on a transport thread (StreamInput in) -> new TransportRequest.Empty(in) { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 5635e07000029..2d0956fae5499 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.SuppressForbidden; @@ -125,7 +126,7 @@ public static MockTransportService startTransport( try { newService.registerRequestHandler( SearchShardsAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, SearchShardsRequest::new, (request, channel, task) -> { if ("index_not_found".equals(request.preference())) { @@ -135,41 +136,50 @@ public static MockTransportService startTransport( } } ); - newService.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { - if ("index_not_found".equals(request.preference())) { - channel.sendResponse(new IndexNotFoundException("index")); - return; - } - SearchHits searchHits; - if ("null_target".equals(request.preference())) { - searchHits = new SearchHits(new SearchHit[] { new SearchHit(0) }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1F); - } else { - searchHits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN); + newService.registerRequestHandler( + SearchAction.NAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SearchRequest::new, + (request, channel, task) -> { + if ("index_not_found".equals(request.preference())) { + channel.sendResponse(new IndexNotFoundException("index")); + return; + } + SearchHits searchHits; + if ("null_target".equals(request.preference())) { + searchHits = new SearchHits( + new SearchHit[] { new SearchHit(0) }, + new TotalHits(1, TotalHits.Relation.EQUAL_TO), + 1F + ); + } else { + searchHits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN); + } + InternalSearchResponse response = new InternalSearchResponse( + searchHits, + InternalAggregations.EMPTY, + null, + null, + false, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + response, + null, + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + channel.sendResponse(searchResponse); } - InternalSearchResponse response = new InternalSearchResponse( - searchHits, - InternalAggregations.EMPTY, - null, - null, - false, - null, - 1 - ); - SearchResponse searchResponse = new SearchResponse( - response, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); - channel.sendResponse(searchResponse); - }); + ); newService.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); @@ -183,7 +193,7 @@ public static MockTransportService startTransport( if (RemoteClusterPortSettings.REMOTE_CLUSTER_SERVER_ENABLED.get(s)) { newService.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(knownNodes)) ); diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java index 577f5d2c7dc18..84e4249bb0ccb 100644 --- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; @@ -120,7 +121,7 @@ public MockTransportService startTransport( try { newService.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); @@ -134,7 +135,7 @@ public MockTransportService startTransport( if (hasClusterCredentials) { newService.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(knownNodes)) ); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java index 704de4fb2276a..32a5b5dec9597 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.RefCounted; @@ -108,32 +109,47 @@ private MockTransportService buildService(VersionInformation version, TransportV } public void testSendMessage() throws InterruptedException { - serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_A"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - }); + serviceA.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_A"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + } + ); final boolean cancellable = randomBoolean(); TransportActionProxy.registerProxyAction(serviceA, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceA, nodeB); - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertThat(task instanceof CancellableTask, equalTo(cancellable)); - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_B"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - }); + serviceB.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertThat(task instanceof CancellableTask, equalTo(cancellable)); + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_B"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + } + ); TransportActionProxy.registerProxyAction(serviceB, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceB, nodeC); - serviceC.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertThat(task instanceof CancellableTask, equalTo(cancellable)); - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_C"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - }); + serviceC.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertThat(task instanceof CancellableTask, equalTo(cancellable)); + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_C"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + } + ); TransportActionProxy.registerProxyAction(serviceC, "internal:test", cancellable, SimpleTestResponse::new); // Node A -> Node B -> Node C: different versions - serialize the response @@ -248,7 +264,7 @@ public void testSendLocalRequest() throws Exception { final boolean cancellable = randomBoolean(); serviceB.registerRequestHandler( "internal:test", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), SimpleTestRequest::new, (request, channel, task) -> { try { @@ -308,24 +324,39 @@ public void handleException(TransportException exp) { public void testException() throws InterruptedException { boolean cancellable = randomBoolean(); - serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - SimpleTestResponse response = new SimpleTestResponse("TS_A"); - channel.sendResponse(response); - }); + serviceA.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + SimpleTestResponse response = new SimpleTestResponse("TS_A"); + channel.sendResponse(response); + } + ); TransportActionProxy.registerProxyAction(serviceA, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceA, nodeB); - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - SimpleTestResponse response = new SimpleTestResponse("TS_B"); - channel.sendResponse(response); - }); + serviceB.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + SimpleTestResponse response = new SimpleTestResponse("TS_B"); + channel.sendResponse(response); + } + ); TransportActionProxy.registerProxyAction(serviceB, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceB, nodeC); - serviceC.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - throw new ElasticsearchException("greetings from TS_C"); - }); + serviceC.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + throw new ElasticsearchException("greetings from TS_C"); + } + ); TransportActionProxy.registerProxyAction(serviceC, "internal:test", cancellable, SimpleTestResponse::new); CountDownLatch latch = new CountDownLatch(1); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java index b6d3ab5455fd6..2eb77c706a3a2 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; @@ -68,7 +69,7 @@ protected void onSendRequest(long requestId, String action, TransportRequest req transportService.registerRequestHandler( testActionName, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, TransportRequest.Empty::new, (request, channel, task) -> channel.sendResponse(TransportResponse.Empty.INSTANCE) ); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java index 6f422190e9870..c5aa918fdc0e2 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java @@ -155,7 +155,7 @@ public ExecutorService executor(String name) { for (final var executor : EXECUTOR_NAMES) { transportService.registerRequestHandler( ACTION_NAME_PREFIX + executor, - executor, + threadPool.executor(executor), TransportRequest.Empty::new, (request, channel, task) -> { if (randomBoolean()) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java b/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java new file mode 100644 index 0000000000000..4edca754df053 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test; + +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockingDetails; +import static org.mockito.Mockito.when; + +/** + * Utilities for setting up Mockito mocks. + */ +public class MockUtils { + + /** + * Sets up a mock TransportService that can answer calls to TransportService.getThreadPool().executor(String). + * + * @return A mocked TransportService instance + */ + public static TransportService setupTransportServiceWithThreadpoolExecutor() { + TransportService transportService = mock(TransportService.class); + ThreadPool threadPool = mock(ThreadPool.class); + + setMockReturns(transportService, threadPool); + + return transportService; + } + + /** + * Sets up a mock TransportService that can answer calls to TransportService.getThreadPool().executor(String), using the given + * threadPool in TransportService. + * + * @param threadPool A mock ThreadPool + * @return A mocked TransportService instance + */ + public static TransportService setupTransportServiceWithThreadpoolExecutor(ThreadPool threadPool) { + assert mockingDetails(threadPool).isMock(); + TransportService transportService = mock(TransportService.class); + + setMockReturns(transportService, threadPool); + + return transportService; + } + + /** + * Sets up the given mock TransportService so that it can answer calls to TransportService.getThreadPool().executor(String). + * + * @param transportService A mock TransportService to be set up. + */ + public static void setupTransportServiceWithThreadpoolExecutor(TransportService transportService) { + assert mockingDetails(transportService).isMock(); + ThreadPool threadPool = mock(ThreadPool.class); + + setMockReturns(transportService, threadPool); + } + + private static void setMockReturns(TransportService transportService, ThreadPool threadPool) { + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index 4a683c3f7dd57..2a80b4e2e7e45 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -313,7 +313,7 @@ public void assertNoPendingHandshakes(Transport transport) { public void testHelloWorld() { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -397,7 +397,7 @@ public void testThreadContext() throws ExecutionException, InterruptedException serviceA.registerRequestHandler( "internal:ping_pong", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertEquals("ping_user", threadPool.getThreadContext().getHeader("test.ping.user")); @@ -460,7 +460,7 @@ public void testLocalNodeConnection() throws InterruptedException { final AtomicReference exception = new AtomicReference<>(); serviceA.registerRequestHandler( "internal:localNode", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { try { @@ -519,8 +519,8 @@ public void testMessageListeners() throws Exception { } }; final String ACTION = "internal:action"; - serviceA.registerRequestHandler(ACTION, ThreadPool.Names.GENERIC, TransportRequest.Empty::new, requestHandler); - serviceB.registerRequestHandler(ACTION, ThreadPool.Names.GENERIC, TransportRequest.Empty::new, requestHandler); + serviceA.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, requestHandler); + serviceB.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, requestHandler); class CountingListener implements TransportMessageListener { AtomicInteger requestsReceived = new AtomicInteger(); @@ -639,7 +639,7 @@ public void testVoidMessageCompressed() throws Exception { try (MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY)) { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, (request, channel, task) -> { try { @@ -695,7 +695,7 @@ public void testHelloWorldCompressed() throws Exception { try (MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY)) { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -765,8 +765,18 @@ public void testIndexingDataCompression() throws Exception { fail(e.getMessage()); } }; - serviceA.registerRequestHandler("internal:sayHello", ThreadPool.Names.GENERIC, StringMessageRequest::new, handler); - serviceC.registerRequestHandler("internal:sayHello", ThreadPool.Names.GENERIC, StringMessageRequest::new, handler); + serviceA.registerRequestHandler( + "internal:sayHello", + threadPool.executor(ThreadPool.Names.GENERIC), + StringMessageRequest::new, + handler + ); + serviceC.registerRequestHandler( + "internal:sayHello", + threadPool.executor(ThreadPool.Names.GENERIC), + StringMessageRequest::new, + handler + ); Settings settingsWithCompress = Settings.builder() .put(TransportSettings.TRANSPORT_COMPRESS.getKey(), Compression.Enabled.INDEXING_DATA) @@ -830,7 +840,7 @@ public void handleException(TransportException exp) { public void testErrorMessage() throws InterruptedException { serviceA.registerRequestHandler( "internal:sayHelloException", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -893,7 +903,7 @@ public void testConcurrentSendRespondAndDisconnect() throws BrokenBarrierExcepti Set responseErrors = ConcurrentCollections.newConcurrentSet(); serviceA.registerRequestHandler( "internal:test", - randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC, + threadPool.executor(randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC), TestRequest::new, (request, channel, task) -> { try { @@ -912,7 +922,7 @@ public void testConcurrentSendRespondAndDisconnect() throws BrokenBarrierExcepti logger.trace("caught exception while responding from node B", e); } }; - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, TestRequest::new, ignoringRequestHandler); + serviceB.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, TestRequest::new, ignoringRequestHandler); int halfSenders = scaledRandomIntBetween(3, 10); final CyclicBarrier go = new CyclicBarrier(halfSenders * 2 + 1); @@ -1003,7 +1013,12 @@ public void onAfter() { // simulate restart of nodeB serviceB.close(); MockTransportService newService = buildService("TS_B_" + i, version1, transportVersion1, Settings.EMPTY); - newService.registerRequestHandler("internal:test", ThreadPool.Names.SAME, TestRequest::new, ignoringRequestHandler); + newService.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + ignoringRequestHandler + ); serviceB = newService; nodeB = newService.getLocalDiscoNode(); connectToNode(serviceB, nodeA); @@ -1027,7 +1042,7 @@ public void testNotifyOnShutdown() throws Exception { try { serviceA.registerRequestHandler( "internal:foobar", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { try { @@ -1060,7 +1075,7 @@ public void testNotifyOnShutdown() throws Exception { public void testTimeoutSendExceptionWithNeverSendingBackResponse() throws Exception { serviceA.registerRequestHandler( "internal:sayHelloTimeoutNoResponse", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> assertThat("moshe", equalTo(request.message)) ); // don't send back a response @@ -1105,7 +1120,7 @@ public void testTimeoutSendExceptionWithDelayedResponse() throws Exception { Semaphore inFlight = new Semaphore(Integer.MAX_VALUE); serviceA.registerRequestHandler( "internal:sayHelloTimeoutDelayedResponse", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { String message = request.message; @@ -1250,12 +1265,22 @@ public void handleResponse(StringMessageResponse response) {} public void handleException(TransportException exp) {} }; - serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceA.registerRequestHandler("internal:testNotSeen", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceA.registerRequestHandler("internal:testError", ThreadPool.Names.SAME, StringMessageRequest::new, handlerWithError); - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceB.registerRequestHandler("internal:testNotSeen", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceB.registerRequestHandler("internal:testError", ThreadPool.Names.SAME, StringMessageRequest::new, handlerWithError); + serviceA.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceA.registerRequestHandler("internal:testNotSeen", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceA.registerRequestHandler( + "internal:testError", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + StringMessageRequest::new, + handlerWithError + ); + serviceB.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceB.registerRequestHandler("internal:testNotSeen", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceB.registerRequestHandler( + "internal:testError", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + StringMessageRequest::new, + handlerWithError + ); String includeSettings; String excludeSettings; @@ -1570,13 +1595,18 @@ public void writeTo(StreamOutput out) throws IOException { } public void testVersionFrom0to1() throws Exception { - serviceB.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version1Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - assertThat(request.value2, equalTo(0)); // not set, coming from service A - Version1Response response = new Version1Response(1, 2); - channel.sendResponse(response); - assertEquals(transportVersion0, channel.getVersion()); - }); + serviceB.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version1Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + assertThat(request.value2, equalTo(0)); // not set, coming from service A + Version1Response response = new Version1Response(1, 2); + channel.sendResponse(response); + assertEquals(transportVersion0, channel.getVersion()); + } + ); Version0Request version0Request = new Version0Request(); version0Request.value1 = 1; @@ -1613,12 +1643,17 @@ public void handleException(TransportException exp) { } public void testVersionFrom1to0() throws Exception { - serviceA.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version0Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - Version0Response response = new Version0Response(1); - channel.sendResponse(response); - assertEquals(transportVersion0, channel.getVersion()); - }); + serviceA.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version0Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + Version0Response response = new Version0Response(1); + channel.sendResponse(response); + assertEquals(transportVersion0, channel.getVersion()); + } + ); Version1Request version1Request = new Version1Request(); version1Request.value1 = 1; @@ -1658,15 +1693,20 @@ public void handleException(TransportException exp) { } public void testVersionFrom1to1() throws Exception { - serviceB.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version1Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - assertThat(request.value2, equalTo(2)); - Version1Response response = new Version1Response(1, 2); - channel.sendResponse(response); - // channel versions don't make sense on DirectResponseChannel - assertThat(channel, instanceOf(TaskTransportChannel.class)); - assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); - }); + serviceB.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version1Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + assertThat(request.value2, equalTo(2)); + Version1Response response = new Version1Response(1, 2); + channel.sendResponse(response); + // channel versions don't make sense on DirectResponseChannel + assertThat(channel, instanceOf(TaskTransportChannel.class)); + assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); + } + ); Version1Request version1Request = new Version1Request(); version1Request.value1 = 1; @@ -1706,14 +1746,19 @@ public void handleException(TransportException exp) { } public void testVersionFrom0to0() throws Exception { - serviceA.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version0Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - Version0Response response = new Version0Response(1); - channel.sendResponse(response); - // channel versions don't make sense on DirectResponseChannel - assertThat(channel, instanceOf(TaskTransportChannel.class)); - assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); - }); + serviceA.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version0Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + Version0Response response = new Version0Response(1); + channel.sendResponse(response); + // channel versions don't make sense on DirectResponseChannel + assertThat(channel, instanceOf(TaskTransportChannel.class)); + assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); + } + ); Version0Request version0Request = new Version0Request(); version0Request.value1 = 1; @@ -1752,7 +1797,7 @@ public void handleException(TransportException exp) { public void testMockFailToSendNoConnectRule() throws Exception { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -1808,7 +1853,7 @@ public void handleException(TransportException exp) { public void testMockUnresponsiveRule() throws InterruptedException { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -1860,11 +1905,16 @@ public void testHostOnMessages() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(2); final AtomicReference addressA = new AtomicReference<>(); final AtomicReference addressB = new AtomicReference<>(); - serviceB.registerRequestHandler("internal:action1", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - addressA.set(request.remoteAddress()); - channel.sendResponse(new TestResponse((String) null)); - latch.countDown(); - }); + serviceB.registerRequestHandler( + "internal:action1", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + addressA.set(request.remoteAddress()); + channel.sendResponse(new TestResponse((String) null)); + latch.countDown(); + } + ); serviceA.sendRequest(nodeB, "internal:action1", new TestRequest(), new TransportResponseHandler() { @Override public TestResponse read(StreamInput in) throws IOException { @@ -1902,10 +1952,15 @@ public void handleException(TransportException exp) { public void testRejectEarlyIncomingRequests() throws Exception { try (TransportService service = buildService("TS_TEST", version0, transportVersion0, null, Settings.EMPTY, false, false)) { AtomicBoolean requestProcessed = new AtomicBoolean(false); - service.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - requestProcessed.set(true); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - }); + service.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + requestProcessed.set(true); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); DiscoveryNode node = service.getLocalNode(); serviceA.close(); @@ -2145,19 +2200,19 @@ public Executor executor(ThreadPool threadPool) { } serviceB.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, new TestRequestHandler(serviceB) ); serviceC.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, new TestRequestHandler(serviceC) ); serviceA.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, new TestRequestHandler(serviceA) ); @@ -2231,7 +2286,7 @@ public Executor executor(ThreadPool threadPool) { public void testRegisterHandlerTwice() { serviceB.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2241,7 +2296,7 @@ public void testRegisterHandlerTwice() { IllegalArgumentException.class, () -> serviceB.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2251,7 +2306,7 @@ public void testRegisterHandlerTwice() { serviceA.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2449,16 +2504,21 @@ public void run() { public void testResponseHeadersArePreserved() throws InterruptedException { List executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet()); CollectionUtil.timSort(executors); // makes sure it's reproducible - serviceA.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { + serviceA.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { - threadPool.getThreadContext().putTransient("boom", new Object()); - threadPool.getThreadContext().addResponseHeader("foo.bar", "baz"); - if ("fail".equals(request.info)) { - throw new RuntimeException("boom"); - } else { - channel.sendResponse(TransportResponse.Empty.INSTANCE); + threadPool.getThreadContext().putTransient("boom", new Object()); + threadPool.getThreadContext().addResponseHeader("foo.bar", "baz"); + if ("fail".equals(request.info)) { + throw new RuntimeException("boom"); + } else { + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } } - }); + ); CountDownLatch latch = new CountDownLatch(2); @@ -2512,9 +2572,14 @@ public void testHandlerIsInvokedOnConnectionClose() throws IOException, Interrup List executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet()); CollectionUtil.timSort(executors); // makes sure it's reproducible TransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); - serviceC.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // do nothing - }); + serviceC.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // do nothing + } + ); CountDownLatch latch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler() { @Override @@ -2576,26 +2641,31 @@ public void testConcurrentDisconnectOnNonPublishedConnection() throws IOExceptio MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); CountDownLatch receivedLatch = new CountDownLatch(1); CountDownLatch sendResponseLatch = new CountDownLatch(1); - serviceC.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); + serviceC.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); + } } - } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - }); - }); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + }); + } + ); CountDownLatch responseLatch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @Override @@ -2643,26 +2713,31 @@ public void testTransportStats() throws Exception { MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); CountDownLatch receivedLatch = new CountDownLatch(1); CountDownLatch sendResponseLatch = new CountDownLatch(1); - serviceB.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); + serviceB.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); + } } - } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - }); - }); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + }); + } + ); CountDownLatch responseLatch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @Override @@ -2755,26 +2830,31 @@ public void testTransportStatsWithException() throws Exception { CountDownLatch sendResponseLatch = new CountDownLatch(1); Exception ex = new RuntimeException("boom"); ex.setStackTrace(new StackTraceElement[0]); - serviceB.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); + serviceB.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); + } } - } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - onFailure(ex); - } - }); - }); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + onFailure(ex); + } + }); + } + ); CountDownLatch responseLatch = new CountDownLatch(1); AtomicReference receivedException = new AtomicReference<>(null); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @@ -3120,31 +3200,41 @@ public void testFailToSendIllegalStateException() throws InterruptedException { public void testChannelToString() { final String ACTION = "internal:action"; - serviceA.registerRequestHandler(ACTION, ThreadPool.Names.SAME, TransportRequest.Empty::new, (request, channel, task) -> { - assertThat( - channel.toString(), - allOf( - containsString("DirectResponseChannel"), - containsString('{' + ACTION + '}'), - containsString("TaskTransportChannel{task=" + task.getId() + '}') - ) - ); - assertThat(new ChannelActionListener<>(channel).toString(), containsString(channel.toString())); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - }); - serviceB.registerRequestHandler(ACTION, ThreadPool.Names.SAME, TransportRequest.Empty::new, (request, channel, task) -> { - assertThat( - channel.toString(), - allOf( - containsString("TcpTransportChannel"), - containsString('{' + ACTION + '}'), - containsString("TaskTransportChannel{task=" + task.getId() + '}'), - containsString("localAddress="), - containsString(serviceB.getLocalNode().getAddress().toString()) - ) - ); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - }); + serviceA.registerRequestHandler( + ACTION, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TransportRequest.Empty::new, + (request, channel, task) -> { + assertThat( + channel.toString(), + allOf( + containsString("DirectResponseChannel"), + containsString('{' + ACTION + '}'), + containsString("TaskTransportChannel{task=" + task.getId() + '}') + ) + ); + assertThat(new ChannelActionListener<>(channel).toString(), containsString(channel.toString())); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); + serviceB.registerRequestHandler( + ACTION, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TransportRequest.Empty::new, + (request, channel, task) -> { + assertThat( + channel.toString(), + allOf( + containsString("TcpTransportChannel"), + containsString('{' + ACTION + '}'), + containsString("TaskTransportChannel{task=" + task.getId() + '}'), + containsString("localAddress="), + containsString(serviceB.getLocalNode().getAddress().toString()) + ) + ); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); PlainActionFuture.get( f -> submitRequest( @@ -3233,7 +3323,7 @@ public void writeTo(StreamOutput out) throws IOException { final var requestSize = between(0, ByteSizeUnit.MB.toIntBytes(1)); final var responseSize = between(0, ByteSizeUnit.MB.toIntBytes(1)); - serviceB.registerRequestHandler(ACTION, ThreadPool.Names.SAME, Request::new, (request, channel, task) -> { + serviceB.registerRequestHandler(ACTION, EsExecutors.DIRECT_EXECUTOR_SERVICE, Request::new, (request, channel, task) -> { assertEquals(requestSize, request.refSize); channel.sendResponse(new Response(responseSize)); }); diff --git a/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java index 2925254ea992b..6d5542db5e529 100644 --- a/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java +++ b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java @@ -322,7 +322,12 @@ public void handleException(TransportException exp) { } private void registerRequestHandler(TransportService transportService, TransportRequestHandler handler) { - transportService.registerRequestHandler(TEST_ACTION, ThreadPool.Names.GENERIC, TestRequest::new, handler); + transportService.registerRequestHandler( + TEST_ACTION, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), + TestRequest::new, + handler + ); } private void send( diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java index ee7924428dd0b..eb7cc9c51c62e 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -59,13 +60,15 @@ public void init() { } public void testAvailable() throws Exception { - AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); Client client = mockClient(); AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, client @@ -84,14 +87,16 @@ public void testAvailable() throws Exception { } public void testEnabled() throws Exception { - AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); assertTrue(featureSet.enabled()); Client client = mockClient(); AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, client diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java index 4f9d803c50c25..6b0b53aedd12d 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; @@ -36,6 +37,7 @@ import static java.util.stream.Collectors.toList; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -44,6 +46,10 @@ public TransportAnalyticsStatsAction action(AnalyticsUsage usage) { TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + ClusterService clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); when(clusterService.localNode()).thenReturn(discoveryNode); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java index 93ac8bdc85896..91dc4f7d9d479 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; @@ -39,10 +40,12 @@ public class TransportDeleteAutoscalingPolicyActionTests extends AutoscalingTestCase { public void testWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportDeleteAutoscalingPolicyAction action = new TransportDeleteAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); @@ -61,10 +64,12 @@ public void testWriteBlock() { } public void testNoWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportDeleteAutoscalingPolicyAction action = new TransportDeleteAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java index 17f97e2ddc050..12d9fcee8bd10 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingLicenseChecker; @@ -37,10 +38,12 @@ public class TransportGetAutoscalingPolicyActionTests extends AutoscalingTestCase { public void testReadBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportGetAutoscalingPolicyAction action = new TransportGetAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), new AutoscalingLicenseChecker(() -> true) @@ -64,10 +67,12 @@ public void testReadBlock() { } public void testNoReadBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportGetAutoscalingPolicyAction action = new TransportGetAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), new AutoscalingLicenseChecker(() -> true) diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java index a39b3113a96ca..bca2fc4ab96af 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingLicenseChecker; @@ -41,10 +42,12 @@ public class TransportPutAutoscalingPolicyActionTests extends AutoscalingTestCas private static final PolicyValidator NO_VALIDATION = policy -> {}; public void testWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportPutAutoscalingPolicyAction action = new TransportPutAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_VALIDATION, @@ -65,10 +68,12 @@ public void testWriteBlock() { } public void testNoWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportPutAutoscalingPolicyAction action = new TransportPutAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_VALIDATION, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java index 7a7d375781e16..1d81549f7adec 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; @@ -48,8 +49,9 @@ public void init() { } public void testAvailable() { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -64,8 +66,9 @@ public void testAvailable() { public void testEnabled() { Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState @@ -73,13 +76,14 @@ public void testEnabled() { assertThat(featureSet.enabled(), equalTo(false)); settings = Settings.builder().put("xpack.ccr.enabled", true); - featureSet = new CCRInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState); + featureSet = new CCRInfoTransportAction(transportService, mock(ActionFilters.class), settings.build(), licenseState); assertThat(featureSet.enabled(), equalTo(true)); } public void testName() { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -137,10 +141,12 @@ public void testUsageStats() throws Exception { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build(); Mockito.when(clusterService.state()).thenReturn(clusterState); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new CCRUsageTransportAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, Settings.EMPTY, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java index 95ab1c2323f7d..f2423ca4d1d16 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; @@ -57,6 +58,7 @@ public void testActionNames() { final TransportService transportService = mock(TransportService.class); final CcrRestoreSourceService ccrRestoreSourceService = mock(CcrRestoreSourceService.class); + MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new ClearCcrRestoreSessionAction.TransportAction(actionFilters, transportService, ccrRestoreSourceService); assertThat(action.actionName, equalTo(ClearCcrRestoreSessionAction.NAME)); @@ -87,6 +89,7 @@ public void testRequestedShardIdMustBeConsistentWithSessionShardId() { } }).when(ccrRestoreSourceService).ensureSessionShardIdConsistency(anyString(), any()); + MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new ClearCcrRestoreSessionAction.TransportAction(actionFilters, transportService, ccrRestoreSourceService); final String sessionUUID = UUIDs.randomBase64UUID(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java index 7357873bba0f2..307bb6e67b10f 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; @@ -62,6 +63,7 @@ public void testActionNames() { final TransportService transportService = mock(TransportService.class); final CcrRestoreSourceService ccrRestoreSourceService = mock(CcrRestoreSourceService.class); + MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new GetCcrRestoreFileChunkAction.TransportAction( bigArrays, transportService, @@ -103,6 +105,7 @@ public void testRequestedShardIdMustBeConsistentWithSessionShardId() { } }).when(ccrRestoreSourceService).ensureSessionShardIdConsistency(anyString(), any()); + MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new GetCcrRestoreFileChunkAction.TransportAction( bigArrays, transportService, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java index 7eca64b82cbb4..1c09fcb746f49 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java @@ -135,7 +135,7 @@ public TransportTermsEnumAction( transportService.registerRequestHandler( transportShardAction, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, NodeTermsEnumRequest::new, new NodeTransportHandler() ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java index 0801ae1834954..b457a77e38f01 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import java.util.ArrayList; @@ -59,8 +60,9 @@ public void testDoExecute() throws Exception { }); } + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); TransportXPackInfoAction action = new TransportXPackInfoAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), licenseService, client diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java index e60f8c1ccae6e..190ac8d388bcf 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ESTestCase; @@ -33,6 +34,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; public class TransportNodeDeprecationCheckActionTests extends ESTestCase { @@ -62,6 +64,11 @@ public void testNodeOperation() { DiscoveryNode node = Mockito.mock(DiscoveryNode.class); when(node.getId()).thenReturn("mock-node"); TransportService transportService = Mockito.mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + when(transportService.getLocalNode()).thenReturn(node); PluginsService pluginsService = Mockito.mock(PluginsService.class); ActionFilters actionFilters = Mockito.mock(ActionFilters.class); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java index 89ff330051418..76577b12cd8d9 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -28,7 +29,8 @@ public class InternalExecutePolicyActionTests extends ESTestCase { @Before public void instantiateTransportAction() { - transportAction = new InternalExecutePolicyAction.Transport(mock(TransportService.class), mock(ActionFilters.class), null, null); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + transportAction = new InternalExecutePolicyAction.Transport(transportService, mock(ActionFilters.class), null, null); } public void testSelectNodeForPolicyExecution() { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java index c350465f0f725..c9d2f97eb5624 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -54,12 +55,14 @@ public void init() throws Exception { } public void testAvailable() { - EqlInfoTransportAction featureSet = new EqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + EqlInfoTransportAction featureSet = new EqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - EqlInfoTransportAction featureSet = new EqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + EqlInfoTransportAction featureSet = new EqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } @@ -93,10 +96,12 @@ public void testUsageStats() throws Exception { when(mockNode.getId()).thenReturn("mocknode"); when(clusterService.localNode()).thenReturn(mockNode); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new EqlUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, client diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java index 8a39bc5110c93..53d5a66de7b66 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java @@ -36,9 +36,9 @@ public class DriverTaskRunner { public static final String ACTION_NAME = "internal:data/read/esql/compute"; private final TransportService transportService; - public DriverTaskRunner(TransportService transportService, String executorName) { + public DriverTaskRunner(TransportService transportService, Executor executor) { this.transportService = transportService; - transportService.registerRequestHandler(ACTION_NAME, executorName, DriverRequest::new, new DriverRequestHandler()); + transportService.registerRequestHandler(ACTION_NAME, executor, DriverRequest::new, new DriverRequestHandler()); } public void executeDrivers(Task parentTask, List drivers, Executor executor, ActionListener listener) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java index c3efd67579d66..6db4a8c4fe37d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java @@ -79,13 +79,13 @@ public ExchangeService(Settings settings, ThreadPool threadPool, String executor public void registerTransportHandler(TransportService transportService) { transportService.registerRequestHandler( EXCHANGE_ACTION_NAME, - requestExecutorName, + threadPool.executor(requestExecutorName), ExchangeRequest::new, new ExchangeTransportAction() ); transportService.registerRequestHandler( OPEN_EXCHANGE_ACTION_NAME, - requestExecutorName, + threadPool.executor(requestExecutorName), OpenExchangeRequest::new, new OpenExchangeRequestHandler() ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 8aff85d088d6f..4d783b9a1012c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -102,12 +102,7 @@ public EnrichLookupService(ClusterService clusterService, SearchService searchSe this.searchService = searchService; this.transportService = transportService; this.executor = transportService.getThreadPool().executor(EsqlPlugin.ESQL_THREAD_POOL_NAME); - transportService.registerRequestHandler( - LOOKUP_ACTION_NAME, - EsqlPlugin.ESQL_THREAD_POOL_NAME, - LookupRequest::new, - new TransportHandler() - ); + transportService.registerRequestHandler(LOOKUP_ACTION_NAME, this.executor, LookupRequest::new, new TransportHandler()); } public void lookupAsync( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index f725e4fe60175..27634bf0d4eaa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -103,14 +103,9 @@ public ComputeService( this.searchService = searchService; this.transportService = transportService; this.bigArrays = bigArrays.withCircuitBreaking(); - transportService.registerRequestHandler( - DATA_ACTION_NAME, - ESQL_THREAD_POOL_NAME, - DataNodeRequest::new, - new DataNodeRequestHandler() - ); this.esqlExecutor = threadPool.executor(ESQL_THREAD_POOL_NAME); - this.driverRunner = new DriverTaskRunner(transportService, ESQL_THREAD_POOL_NAME); + transportService.registerRequestHandler(DATA_ACTION_NAME, this.esqlExecutor, DataNodeRequest::new, new DataNodeRequestHandler()); + this.driverRunner = new DriverTaskRunner(transportService, this.esqlExecutor); this.exchangeService = exchangeService; this.enrichLookupService = enrichLookupService; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java index 7f0bdbab6add3..f0b3a89a4444b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java @@ -43,11 +43,17 @@ public class EsqlInfoTransportActionTests extends ESTestCase { private ThreadPool threadPool; + private TransportService transportService; private Client client; @Before public void init() { threadPool = new TestThreadPool(getTestName()); + + // TODO: temporary, remove in #97879 + transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); } @@ -58,12 +64,12 @@ public void shutdown() { } public void testAvailable() { - EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } @@ -98,7 +104,7 @@ public void testUsageStats() throws Exception { when(clusterService.localNode()).thenReturn(mockNode); var usageAction = new EsqlUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, threadPool, mock(ActionFilters.class), diff --git a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java b/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java index f1bad49abda68..90034c9620923 100644 --- a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java +++ b/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -33,8 +34,9 @@ public void init() throws Exception { } public void testAvailable() throws Exception { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GraphInfoTransportAction featureSet = new GraphInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -44,7 +46,7 @@ public void testAvailable() throws Exception { assertThat(featureSet.available(), is(available)); var usageAction = new GraphUsageTransportAction( - mock(TransportService.class), + transportService, null, mock(ThreadPool.class), mock(ActionFilters.class), @@ -73,8 +75,10 @@ public void testEnabled() throws Exception { } else { settings.put("xpack.graph.enabled", enabled); } + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GraphInfoTransportAction featureSet = new GraphInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState @@ -82,7 +86,7 @@ public void testEnabled() throws Exception { assertThat(featureSet.enabled(), is(enabled)); GraphUsageTransportAction usageAction = new GraphUsageTransportAction( - mock(TransportService.class), + transportService, null, mock(ThreadPool.class), mock(ActionFilters.class), diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java index ae031a050d3d5..0ea85bab36a4e 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider; import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument; @@ -57,10 +58,10 @@ public void setupMocks() { index = mock(SamlServiceProviderIndex.class); idp = mock(SamlIdentityProvider.class); when(idp.getAllowedNameIdFormats()).thenReturn(Set.of(TRANSIENT)); - now = Instant.ofEpochMilli(System.currentTimeMillis() + randomLongBetween(-500_000, 500_000)); final Clock clock = Clock.fixed(now, randomZone()); - action = new TransportPutSamlServiceProviderAction(mock(TransportService.class), mock(ActionFilters.class), index, idp, clock); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + action = new TransportPutSamlServiceProviderAction(transportService, mock(ActionFilters.class), index, idp, clock); } public void testRegisterNewServiceProvider() throws Exception { diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java index 107421ce12a31..d81faf6a398d7 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; @@ -52,18 +53,14 @@ public void init() throws Exception { } public void testAvailable() { - IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), equalTo(true)); } public void testName() { - IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.name(), equalTo("ilm")); } @@ -94,13 +91,9 @@ public void testUsageStats() throws Exception { ClusterState clusterState = buildClusterState(policies, indexPolicies); Mockito.when(clusterService.state()).thenReturn(clusterState); - var usageAction = new IndexLifecycleUsageTransportAction( - mock(TransportService.class), - null, - mock(ThreadPool.class), - mock(ActionFilters.class), - null - ); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + var usageAction = new IndexLifecycleUsageTransportAction(transportService, null, threadPool, mock(ActionFilters.class), null); PlainActionFuture future = new PlainActionFuture<>(); usageAction.masterOperation(null, null, clusterState, future); IndexLifecycleFeatureSetUsage ilmUsage = (IndexLifecycleFeatureSetUsage) future.get().getUsage(); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java index feb66c12c2613..3e0f19246dfe3 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction; @@ -20,10 +21,12 @@ public class TransportDeleteLifecycleActionTests extends ESTestCase { public void testReservedStateHandler() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportDeleteLifecycleAction putAction = new TransportDeleteLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java index ef1d65f2e4950..8467e46630876 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -46,10 +47,12 @@ public void testIsNoop() { } public void testReservedStateHandler() throws Exception { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportPutLifecycleAction putAction = new TransportPutLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(NamedXContentRegistry.class), diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java index cdc50f28fc306..2c19a79ac5af0 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.StopILMRequest; @@ -34,10 +35,12 @@ public class TransportStopILMActionTests extends ESTestCase { public void testStopILMClusterStatePriorityIsImmediate() { ClusterService clusterService = mock(ClusterService.class); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportStopILMAction transportStopILMAction = new TransportStopILMAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java index d72d079d4ae19..56b7965be3686 100644 --- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java +++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportService; @@ -27,8 +28,9 @@ public class TransportDeletePipelineActionTests extends ESTestCase { public void testDeletePipelineWithMissingIndex() throws Exception { try (Client client = getFailureClient(new IndexNotFoundException("missing .logstash"))) { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportDeletePipelineAction action = new TransportDeletePipelineAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), client ); diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java index d8b4b167bcc46..d8a4d048f1fe4 100644 --- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java +++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLogAppender; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportService; @@ -100,11 +101,8 @@ public void onFailure(Exception e) { try (Client client = getMockClient(multiGetResponse)) { Loggers.addAppender(logger, mockLogAppender); - TransportGetPipelineAction action = new TransportGetPipelineAction( - mock(TransportService.class), - mock(ActionFilters.class), - client - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + TransportGetPipelineAction action = new TransportGetPipelineAction(transportService, mock(ActionFilters.class), client); action.doExecute(null, request, testActionListener); } finally { Loggers.removeAppender(logger, mockLogAppender); @@ -152,8 +150,9 @@ public void onFailure(Exception e) { } }; + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); try (Client client = getMockClient(searchResponse)) { - new TransportGetPipelineAction(mock(TransportService.class), mock(ActionFilters.class), client).doExecute( + new TransportGetPipelineAction(transportService, mock(ActionFilters.class), client).doExecute( null, request, testActionListener @@ -165,8 +164,9 @@ public void onFailure(Exception e) { public void testMissingIndexHandling() throws Exception { try (Client failureClient = getFailureClient(new IndexNotFoundException("foo"))) { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportGetPipelineAction action = new TransportGetPipelineAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), failureClient ); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java index 418d277977c16..2ff9bb2906a5c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.ingest.IngestStats; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -145,10 +146,12 @@ private MachineLearningUsageTransportAction newUsageAction( boolean isDataFrameAnalyticsEnabled, boolean isNlpEnabled ) { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); return new MachineLearningUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), TestEnvironment.newEnvironment(settings), @@ -162,8 +165,9 @@ private MachineLearningUsageTransportAction newUsageAction( } public void testAvailable() throws Exception { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); MachineLearningInfoTransportAction featureSet = new MachineLearningInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), commonSettings, licenseState @@ -191,9 +195,11 @@ public void testEnabled() throws Exception { enabled = randomBoolean(); settings.put("xpack.ml.enabled", enabled); } + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); boolean expected = enabled; MachineLearningInfoTransportAction featureSet = new MachineLearningInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java index c55b78c07939a..67da449af850f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.MlTasks; @@ -337,10 +338,12 @@ public static void addTask( } private TransportCloseJobAction createAction() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); return new TransportCloseJobAction( - mock(TransportService.class), + transportService, client, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), clusterService, mock(AnomalyDetectionAuditor.class), diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java index f85a13fcb3935..26409b1ada849 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java @@ -38,6 +38,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; public class TransportDeleteExpiredDataActionTests extends ESTestCase { @@ -59,6 +60,10 @@ public void remove(float requestsPerSec, ActionListener listener, Boole public void setup() { threadPool = new TestThreadPool("TransportDeleteExpiredDataActionTests thread pool"); TransportService transportService = mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + Client client = mock(Client.class); ClusterService clusterService = mock(ClusterService.class); auditor = mock(AnomalyDetectionAuditor.class); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java index 1c0243fcab098..c8096a7b1f384 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java @@ -81,8 +81,12 @@ public void testOperation() { } private TransportFinalizeJobExecutionAction createAction(ClusterService clusterService) { + // TODO: temporary, remove in #97879 + TransportService transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + return new TransportFinalizeJobExecutionAction( - mock(TransportService.class), + transportService, clusterService, threadPool, mock(ActionFilters.class), diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java index 79a376b90040f..4d242db394d10 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; @@ -44,18 +45,14 @@ public class MonitoringInfoTransportActionTests extends ESTestCase { private final Exporters exporters = mock(Exporters.class); public void testAvailable() { - MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testMonitoringEnabledByDefault() { - MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } @@ -94,10 +91,12 @@ public void testUsage() throws Exception { when(exporters.getEnabledExporters()).thenReturn(exporterList); when(monitoring.isMonitoringActive()).thenReturn(collectionEnabled); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new MonitoringUsageTransportAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, new MonitoringUsageServices(monitoring, exporters) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java index 3676fc5d2e7a0..d2a694c04ae34 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -98,6 +99,10 @@ public void setUpMocks() { transportService = mock(TransportService.class); filters = mock(ActionFilters.class); + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + when(transportService.getTaskManager()).thenReturn(taskManager); when(taskManager.register(anyString(), eq(MonitoringBulkAction.NAME), any(TaskAwareRequest.class))).thenReturn(mock(Task.class)); when(filters.filters()).thenReturn(new ActionFilter[0]); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java index dd78eb95e7f3d..9fe634a178179 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -49,7 +50,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportService; @@ -105,7 +105,7 @@ public TransportRollupSearchAction( transportService.registerRequestHandler( actionName, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, true, SearchRequest::new, diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java index ab7420f0b50ee..b0881eb350d5a 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -25,23 +26,21 @@ public class RollupInfoTransportActionTests extends ESTestCase { public void testAvailable() { - RollupInfoTransportAction featureSet = new RollupInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + RollupInfoTransportAction featureSet = new RollupInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabledDefault() { - RollupInfoTransportAction featureSet = new RollupInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + RollupInfoTransportAction featureSet = new RollupInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } public void testUsage() throws ExecutionException, InterruptedException, IOException { - var usageAction = new RollupUsageTransportAction( - mock(TransportService.class), - null, - mock(ThreadPool.class), - mock(ActionFilters.class), - null - ); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + var usageAction = new RollupUsageTransportAction(transportService, null, threadPool, mock(ActionFilters.class), null); PlainActionFuture future = new PlainActionFuture<>(); usageAction.masterOperation(null, null, null, future); XPackFeatureSet.Usage rollupUsage = future.get().getUsage(); diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java index ddba0395382f3..985bec3ba8c77 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -1108,7 +1109,7 @@ private static MockTransportService startTransport( try { service.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { capturedHeaders.add( @@ -1121,7 +1122,7 @@ private static MockTransportService startTransport( ); service.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> { capturedHeaders.add( @@ -1132,7 +1133,7 @@ private static MockTransportService startTransport( ); service.registerRequestHandler( SearchShardsAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, SearchShardsRequest::new, (request, channel, task) -> { capturedHeaders.add( @@ -1141,31 +1142,36 @@ private static MockTransportService startTransport( channel.sendResponse(new SearchShardsResponse(List.of(), List.of(), Collections.emptyMap())); } ); - service.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { - capturedHeaders.add( - new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders())) - ); - channel.sendResponse( - new SearchResponse( - new InternalSearchResponse( - new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - null, - false, + service.registerRequestHandler( + SearchAction.NAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SearchRequest::new, + (request, channel, task) -> { + capturedHeaders.add( + new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders())) + ); + channel.sendResponse( + new SearchResponse( + new InternalSearchResponse( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), + InternalAggregations.EMPTY, + null, + null, + false, + null, + 1 + ), null, - 1 - ), - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) - ); - }); + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ) + ); + } + ); service.start(); service.acceptIncomingRequests(); success = true; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java index 4b750fb7d2ab6..88f233087e1dd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ToXContent; @@ -78,24 +79,18 @@ public void init() throws Exception { } public void testAvailable() { - SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class), - settings - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), settings); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class), - settings - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), settings); assertThat(featureSet.enabled(), is(true)); Settings disabled = Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), false).build(); - featureSet = new SecurityInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class), disabled); + featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), disabled); assertThat(featureSet.enabled(), is(false)); } @@ -371,10 +366,12 @@ private void configureRoleMappingStoreUsage(boolean roleMappingStoreEnabled) { } private SecurityUsageTransportAction newUsageAction(Settings settings) { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); return new SecurityUsageTransportAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, settings, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java index e2d1fc3a24568..f94acab50b6b5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyResponse; @@ -42,12 +43,8 @@ public void setUp() throws Exception { super.setUp(); apiKeyService = mock(ApiKeyService.class); securityContext = mock(SecurityContext.class); - action = new TransportCreateCrossClusterApiKeyAction( - mock(TransportService.class), - mock(ActionFilters.class), - apiKeyService, - securityContext - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + action = new TransportCreateCrossClusterApiKeyAction(transportService, mock(ActionFilters.class), apiKeyService, securityContext); } public void testApiKeyWillBeCreatedWithEmptyUserRoleDescriptors() throws IOException { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java index 5d68dea7a146e..f9e219f168718 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java @@ -61,6 +61,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; public class TransportGrantApiKeyActionTests extends ESTestCase { @@ -69,6 +70,7 @@ public class TransportGrantApiKeyActionTests extends ESTestCase { private ApiKeyUserRoleDescriptorResolver resolver; private AuthenticationService authenticationService; private ThreadPool threadPool; + private TransportService transportService; private AuthorizationService authorizationService; @Before @@ -81,8 +83,12 @@ public void setupMocks() throws Exception { threadPool = new TestThreadPool("TP-" + getTestName()); final ThreadContext threadContext = threadPool.getThreadContext(); + // TODO: temporary, remove in #97879 + TransportService transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + action = new TransportGrantApiKeyAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), threadContext, authenticationService, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java index 711c604206823..7ce920506d7d1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.apikey.ApiKey; @@ -55,8 +56,9 @@ public void testExecute() throws IOException { () -> AuthenticationTestHelper.builder().build() ); when(securityContext.getAuthentication()).thenReturn(authentication); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportUpdateCrossClusterApiKeyAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), apiKeyService, securityContext @@ -120,8 +122,9 @@ public void testExecute() throws IOException { public void testAuthenticationCheck() { final SecurityContext securityContext = mock(SecurityContext.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportUpdateCrossClusterApiKeyAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), mock(ApiKeyService.class), securityContext diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java index ed01d0ca6fc40..dca8633d7c1eb 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java @@ -64,6 +64,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportProfileHasPrivilegesActionTests extends ESTestCase { @@ -80,6 +81,10 @@ public class TransportProfileHasPrivilegesActionTests extends ESTestCase { public void setup() { threadPool = new TestThreadPool(TransportProfileHasPrivilegesActionTests.class.getSimpleName()); transportService = mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + actionFilters = mock(ActionFilters.class); authorizationService = mock(AuthorizationService.class); nativePrivilegeStore = mock(NativePrivilegeStore.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java index df414207da8ee..92bef206a9d6a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest; @@ -50,10 +51,13 @@ public void setup() { fileRealm = mockRealm("file"); final Realms realms = mockRealms(List.of(nativeRealm, fileRealm)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + action = new TransportClearRealmCacheAction( - mock(ThreadPool.class), + threadPool, mockClusterService(), - mock(TransportService.class), + transportService, mock(ActionFilters.class), realms, authenticationService diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java index ac7680ea2b574..5194ff032c5ba 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.service.CreateServiceAccountTokenRequest; @@ -40,8 +41,10 @@ public class TransportCreateServiceAccountTokenActionTests extends ESTestCase { public void init() throws IOException { serviceAccountService = mock(ServiceAccountService.class); securityContext = mock(SecurityContext.class); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportCreateServiceAccountTokenAction = new TransportCreateServiceAccountTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), serviceAccountService, securityContext diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java index 8b0530e48c560..2050d31638fe9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.DeleteServiceAccountTokenRequest; import org.elasticsearch.xpack.core.security.action.service.DeleteServiceAccountTokenResponse; @@ -32,8 +33,9 @@ public class TransportDeleteServiceAccountTokenActionTests extends ESTestCase { @Before public void init() { serviceAccountService = mock(ServiceAccountService.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportDeleteServiceAccountTokenAction = new TransportDeleteServiceAccountTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), serviceAccountService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java index 87ef1ea24559e..e8d178efe6b4a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountRequest; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountResponse; @@ -31,8 +32,9 @@ public class TransportGetServiceAccountActionTests extends ESTestCase { @Before public void init() { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportGetServiceAccountAction = new TransportGetServiceAccountAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()) ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java index da6508ac79ef5..ec68ed7a2776f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountCredentialsRequest; @@ -58,8 +59,10 @@ public void init() throws UnknownHostException { when(transport.boundAddress()).thenReturn(new BoundTransportAddress(new TransportAddress[] { transportAddress }, transportAddress)); final Settings settings = builder.build(); serviceAccountService = mock(ServiceAccountService.class); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportGetServiceAccountCredentialsAction = new TransportGetServiceAccountCredentialsAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), serviceAccountService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java index bd63dd65824bd..dcdeace6b06b6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java @@ -88,6 +88,7 @@ public class TransportCreateTokenActionTests extends ESTestCase { .build(); private ThreadPool threadPool; + private TransportService transportService; private Client client; private SecurityIndexManager securityIndex; private ClusterService clusterService; @@ -99,6 +100,11 @@ public class TransportCreateTokenActionTests extends ESTestCase { @Before public void setupClient() { threadPool = new TestThreadPool(getTestName()); + + // TODO: temporary, remove in #97879 + transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + client = mock(Client.class); idxReqReference = new AtomicReference<>(); authenticationService = mock(AuthenticationService.class); @@ -221,7 +227,7 @@ public void testClientCredentialsCreatesWithoutRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -262,7 +268,7 @@ public void testPasswordGrantTypeCreatesWithRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -305,7 +311,7 @@ public void testKerberosGrantTypeCreatesWithRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -358,7 +364,7 @@ public void testKerberosGrantTypeWillFailOnBase64DecodeError() throws Exception final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -397,7 +403,7 @@ public void testServiceAccountCannotCreateOAuthToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java index a24361078ff26..f07bb7dc365ae 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java @@ -56,6 +56,7 @@ public class TransportInvalidateTokenActionTests extends ESTestCase { .build(); private ThreadPool threadPool; + private TransportService transportService; private Client client; private SecurityIndexManager securityIndex; private ClusterService clusterService; @@ -65,6 +66,11 @@ public class TransportInvalidateTokenActionTests extends ESTestCase { @Before public void setup() { threadPool = new TestThreadPool(getTestName()); + + // TODO: temporary, remove in #97879 + transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + securityContext = new SecurityContext(Settings.EMPTY, threadPool.getThreadContext()); client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); @@ -91,7 +97,7 @@ public void testInvalidateTokensWhenIndexUnavailable() throws Exception { clusterService ); final TransportInvalidateTokenAction action = new TransportInvalidateTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService ); @@ -137,7 +143,7 @@ public void testInvalidateTokensWhenIndexClosed() throws Exception { clusterService ); final TransportInvalidateTokenAction action = new TransportInvalidateTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java index b4a79e070e927..9f7815a1c9891 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest; @@ -57,8 +58,10 @@ public void testHasPrivilegesRequestDoesNotAllowDLSRoleQueryBasedIndicesPrivileg .build(false); when(context.getAuthentication()).thenReturn(authentication); threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportHasPrivilegesAction transportHasPrivilegesAction = new TransportHasPrivilegesAction( - mock(TransportService.class), + transportService, new ActionFilters(Set.of()), mock(AuthorizationService.class), mock(NativePrivilegeStore.class), @@ -110,8 +113,10 @@ public void testRequiresSameUser() { ); return null; }).when(authorizationService).checkPrivileges(any(), any(), anyCollection(), anyActionListener()); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportHasPrivilegesAction( - mock(TransportService.class), + transportService, new ActionFilters(Set.of()), authorizationService, privilegeStore, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java index 600d4d1ba5fa4..2a15aa09ddccd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.IOUtils; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.mocksocket.MockSocket; @@ -159,7 +160,7 @@ public TransportRequestHandler interceptHandler( DiscoveryNode remoteNode = remoteTransportService.getLocalDiscoNode(); remoteTransportService.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(List.of(remoteNode))) ); diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java index 67ce8c255fd94..cf28bf9922b24 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java @@ -19,8 +19,8 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.shutdown.TransportDeleteShutdownNodeAction.DeleteShutdownNodeExecutor; import org.elasticsearch.xpack.shutdown.TransportDeleteShutdownNodeAction.DeleteShutdownNodeTask; import org.junit.Before; @@ -55,7 +55,7 @@ public void init() { MockitoAnnotations.openMocks(this); // TODO: it takes almost 2 seconds to create these mocks....WHY?!? var threadPool = mock(ThreadPool.class); - var transportService = mock(TransportService.class); + var transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); clusterService = mock(ClusterService.class); var actionFilters = mock(ActionFilters.class); var indexNameExpressionResolver = mock(IndexNameExpressionResolver.class); diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java index c264ad6ad1d03..cbd51ceebc729 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java @@ -19,8 +19,8 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.shutdown.TransportPutShutdownNodeAction.PutShutdownNodeExecutor; import org.elasticsearch.xpack.shutdown.TransportPutShutdownNodeAction.PutShutdownNodeTask; import org.junit.Before; @@ -61,7 +61,7 @@ public void init() { MockitoAnnotations.openMocks(this); // TODO: it takes almost 2 seconds to create these mocks....WHY?!? var threadPool = mock(ThreadPool.class); - var transportService = mock(TransportService.class); + var transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); clusterService = mock(ClusterService.class); var actionFilters = mock(ActionFilters.class); var indexNameExpressionResolver = mock(IndexNameExpressionResolver.class); diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java index 4c71683a972a1..954b5ba024418 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.reservedstate.service.ReservedStateUpdateTask; import org.elasticsearch.reservedstate.service.ReservedStateUpdateTaskExecutor; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentParser; @@ -363,10 +364,12 @@ public void testOperatorControllerFromJSONContent() throws IOException { } public void testDeleteSLMReservedStateHandler() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var deleteAction = new TransportDeleteSnapshotLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); @@ -377,10 +380,12 @@ public void testDeleteSLMReservedStateHandler() { } public void testPutSLMReservedStateHandler() throws Exception { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putAction = new TransportPutSnapshotLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java index 0294c14f57494..5d0e002c49036 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.action.StopILMAction; @@ -33,11 +34,12 @@ public class TransportStopSLMActionTests extends ESTestCase { public void testStopILMClusterStatePriorityIsImmediate() { ClusterService clusterService = mock(ClusterService.class); - + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportStopSLMAction transportStopSLMAction = new TransportStopSLMAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java index 83eb9d49a4e21..ede64decbca33 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -56,13 +57,15 @@ public void init() { } public void testAvailable() throws Exception { - SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); var usageAction = new SpatialUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, mockClient() @@ -80,14 +83,16 @@ public void testAvailable() throws Exception { } public void testEnabled() throws Exception { - SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); assertTrue(featureSet.enabled()); SpatialUsageTransportAction usageAction = new SpatialUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, mockClient() diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java index 4f42305984b57..1b91f686a7d8d 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -47,8 +48,8 @@ public class SpatialStatsTransportActionTests extends ESTestCase { @Before public void mockServices() { - transportService = mock(TransportService.class); threadPool = mock(ThreadPool.class); + transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java index f95fd337fe5b7..01bad2c3dee60 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -57,11 +58,8 @@ public void init() throws Exception { } public void testAvailable() { - SqlInfoTransportAction featureSet = new SqlInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class), - licenseState - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + SqlInfoTransportAction featureSet = new SqlInfoTransportAction(transportService, mock(ActionFilters.class), licenseState); assertThat(featureSet.available(), is(true)); } @@ -95,10 +93,12 @@ public void testUsageStats() throws Exception { when(mockNode.getId()).thenReturn("mocknode"); when(clusterService.localNode()).thenReturn(mockNode); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new SqlUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, licenseState, diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java index 5c6f875415384..d2c23f768cef6 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; @@ -29,12 +30,14 @@ public class TransformInfoTransportActionTests extends ESTestCase { public void testAvailable() { - TransformInfoTransportAction featureSet = new TransformInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + TransformInfoTransportAction featureSet = new TransformInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabledDefault() { - TransformInfoTransportAction featureSet = new TransformInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + TransformInfoTransportAction featureSet = new TransformInfoTransportAction(transportService, mock(ActionFilters.class)); assertTrue(featureSet.enabled()); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java index 0b796ee3d3f7d..ac683dca5bf26 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -55,20 +56,23 @@ public class WatcherInfoTransportActionTests extends ESTestCase { private MockLicenseState licenseState; private Client client; + private ThreadPool threadPool; + private TransportService transportService; @Before public void init() throws Exception { licenseState = mock(MockLicenseState.class); client = mock(Client.class); - ThreadPool threadPool = mock(ThreadPool.class); + threadPool = mock(ThreadPool.class); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); when(threadPool.getThreadContext()).thenReturn(threadContext); when(client.threadPool()).thenReturn(threadPool); + transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); } public void testAvailable() { WatcherInfoTransportAction featureSet = new WatcherInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -89,7 +93,7 @@ public void testEnabled() { settings.put("xpack.watcher.enabled", enabled); } WatcherInfoTransportAction featureSet = new WatcherInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState @@ -130,9 +134,9 @@ public void testUsageStats() throws Exception { when(clusterService.localNode()).thenReturn(mockNode); var usageAction = new WatcherUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, Settings.EMPTY, diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java index 79c93cd5b913a..338f4e56f5663 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.license.TestUtils; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.watcher.WatcherMetadata; @@ -56,8 +57,8 @@ public class TransportAckWatchActionTests extends ESTestCase { @Before public void setupAction() { - TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); when(threadPool.getThreadContext()).thenReturn(threadContext); WatchParser watchParser = mock(WatchParser.class); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java index 09cf9daaf3685..95272619fac5d 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.license.TestUtils; import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ClientHelper; @@ -63,7 +64,7 @@ public void setupAction() throws Exception { ThreadPool threadPool = mock(ThreadPool.class); when(threadPool.getThreadContext()).thenReturn(threadContext); - TransportService transportService = mock(TransportService.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); WatchParser parser = mock(WatchParser.class); when(parser.parseWithSecrets(eq("_id"), eq(false), any(), any(), any(), anyBoolean(), anyLong(), anyLong())).thenReturn(watch); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java index 8d07d1eaf6043..6dfdf47a7a735 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -44,8 +45,8 @@ public class TransportWatcherStatsActionTests extends ESTestCase { @Before public void setupTransportAction() { - TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); ClusterService clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); From fe31e92020272e372c719b31a8abd51c206f8132 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 8 Sep 2023 10:23:15 -0700 Subject: [PATCH 47/69] Fix example referencing TransportVersion constant (#99375) With the recent move of constants to the TransportVersions class, the example plugins were missed. This commit fixes the example plugins which referenced a constant. --- .../example/customsuggester/CustomSuggestionBuilder.java | 3 ++- .../elasticsearch/example/rescore/ExampleRescoreBuilder.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java index 827f3e837bdef..298ddb7253eec 100644 --- a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java +++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java @@ -9,6 +9,7 @@ package org.elasticsearch.example.customsuggester; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -132,6 +133,6 @@ public SuggestionSearchContext.SuggestionContext build(SearchExecutionContext co @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersion.ZERO; + return TransportVersions.ZERO; } } diff --git a/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java b/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java index 8691ab4e4f6e4..c54d1b1892d4d 100644 --- a/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java +++ b/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java @@ -14,6 +14,7 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; @@ -218,6 +219,6 @@ public Explanation explain(int topLevelDocId, IndexSearcher searcher, RescoreCon @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersion.ZERO; + return TransportVersions.ZERO; } } From f1a376c317b5b184c4793b1e3ab8b6acabee2bab Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 8 Sep 2023 19:24:31 +0200 Subject: [PATCH 48/69] Remove CopyTo.Builder (#99368) The copyTo builder is really hard to reason about when it comes to mapper merging, because the `reset` method would actually mutate an existing mapper. That seems dangerous and the whole thing is quite inefficient as well. -> this PR just removes it and uses a copy constructor for copy on write, avoiding instance creation on mapper merges here and there and leaving no doubt about these things being immutable. --- .../mapper/LegacyGeoShapeFieldMapper.java | 2 +- .../extras/MatchOnlyTextFieldMapper.java | 10 +---- .../mapper/extras/RankFeatureFieldMapper.java | 2 +- .../extras/RankFeaturesFieldMapper.java | 2 +- .../mapper/extras/ScaledFloatFieldMapper.java | 2 +- .../extras/SearchAsYouTypeFieldMapper.java | 2 +- .../mapper/extras/TokenCountFieldMapper.java | 2 +- .../percolator/PercolatorFieldMapper.java | 2 +- .../icu/ICUCollationKeywordFieldMapper.java | 2 +- .../AnnotatedTextFieldMapper.java | 2 +- .../mapper/murmur3/Murmur3FieldMapper.java | 2 +- .../index/mapper/BinaryFieldMapper.java | 2 +- .../index/mapper/BooleanFieldMapper.java | 9 +--- .../index/mapper/CompletionFieldMapper.java | 2 +- .../index/mapper/DateFieldMapper.java | 2 +- .../index/mapper/FieldMapper.java | 44 ++++++------------- .../index/mapper/GeoPointFieldMapper.java | 2 +- .../index/mapper/GeoShapeFieldMapper.java | 2 +- .../index/mapper/IpFieldMapper.java | 2 +- .../index/mapper/KeywordFieldMapper.java | 2 +- .../index/mapper/NumberFieldMapper.java | 9 +--- .../index/mapper/PlaceHolderFieldMapper.java | 8 +--- .../index/mapper/RangeFieldMapper.java | 2 +- .../index/mapper/TextFieldMapper.java | 2 +- .../flattened/FlattenedFieldMapper.java | 1 - .../vectors/DenseVectorFieldMapper.java | 2 +- .../vectors/SparseVectorFieldMapper.java | 2 +- .../index/mapper/ParametrizedMapperTests.java | 2 +- .../index/mapper/MockFieldMapper.java | 5 ++- .../mapper/HistogramFieldMapper.java | 2 +- .../unsignedlong/UnsignedLongFieldMapper.java | 2 +- .../VersionStringFieldMapper.java | 2 +- .../GeoShapeWithDocValuesFieldMapper.java | 2 +- .../index/mapper/PointFieldMapper.java | 2 +- .../index/mapper/ShapeFieldMapper.java | 2 +- .../wildcard/mapper/WildcardFieldMapper.java | 2 +- 36 files changed, 49 insertions(+), 95 deletions(-) diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java index 1a7f1a2a82883..9b16a08d4ba2f 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java @@ -363,7 +363,7 @@ private static int getLevels(int treeLevels, double precisionInMeters, int defau public LegacyGeoShapeFieldMapper build(MapperBuilderContext context) { LegacyGeoShapeParser parser = new LegacyGeoShapeParser(); GeoShapeFieldType ft = buildFieldType(parser, context); - return new LegacyGeoShapeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), parser, this); + return new LegacyGeoShapeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index d89c5db66f37b..3b54668a91423 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -136,15 +136,7 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) { public MatchOnlyTextFieldMapper build(MapperBuilderContext context) { MatchOnlyTextFieldType tft = buildFieldType(context); MultiFields multiFields = multiFieldsBuilder.build(this, context); - return new MatchOnlyTextFieldMapper( - name, - Defaults.FIELD_TYPE, - tft, - multiFields, - copyTo.build(), - context.isSourceSynthetic(), - this - ); + return new MatchOnlyTextFieldMapper(name, Defaults.FIELD_TYPE, tft, multiFields, copyTo, context.isSourceSynthetic(), this); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index f277450aa1833..b021f5b8e6954 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -110,7 +110,7 @@ public RankFeatureFieldMapper build(MapperBuilderContext context) { nullValue.getValue() ), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, positiveScoreImpact.getValue(), nullValue.getValue() ); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java index 1777a9ac4ba66..f36dfb5605633 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java @@ -67,7 +67,7 @@ public RankFeaturesFieldMapper build(MapperBuilderContext context) { name, new RankFeaturesFieldType(context.buildFullName(name), meta.getValue(), positiveScoreImpact.getValue()), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, positiveScoreImpact.getValue() ); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index bd45ff7a19bfe..43951878933fa 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -195,7 +195,7 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) { metric.getValue(), indexMode ); - return new ScaledFloatFieldMapper(name, type, multiFieldsBuilder.build(this, context), copyTo.build(), this); + return new ScaledFloatFieldMapper(name, type, multiFieldsBuilder.build(this, context), copyTo, this); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java index 4a7bf149315a1..bb8b7ad382044 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java @@ -262,7 +262,7 @@ public SearchAsYouTypeFieldMapper build(MapperBuilderContext context) { return new SearchAsYouTypeFieldMapper( name, ft, - copyTo.build(), + copyTo, indexAnalyzers, prefixFieldMapper, shingleFieldMappers, diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index ced2b900f5a16..12829ca802425 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -87,7 +87,7 @@ public TokenCountFieldMapper build(MapperBuilderContext context) { nullValue.getValue(), meta.getValue() ); - return new TokenCountFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this); + return new TokenCountFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, this); } } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index f5b3f06793183..d2e30ed1d420d 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -163,7 +163,7 @@ public PercolatorFieldMapper build(MapperBuilderContext context) { name(), fieldType, multiFields, - copyTo.build(), + copyTo, searchExecutionContext, extractedTermsField, extractionResultField, diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 9d58549763b62..d0c20e6aceabf 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -341,7 +341,7 @@ public ICUCollationKeywordFieldMapper build(MapperBuilderContext context) { buildFieldType(), ft, multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, collator, this ); diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index ec98bd163d635..2cd46ee170c08 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -148,7 +148,7 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) { fieldType, buildFieldType(fieldType, context), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, this ); } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index b7957ab7e8137..97bed0c5e06a3 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -68,7 +68,7 @@ public Murmur3FieldMapper build(MapperBuilderContext context) { name, new Murmur3FieldType(context.buildFullName(name), stored.getValue(), meta.getValue()), multiFieldsBuilder.build(this, context), - copyTo.build() + copyTo ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index f5d2dccab8052..98aaf3fbda596 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -67,7 +67,7 @@ public BinaryFieldMapper build(MapperBuilderContext context) { name, new BinaryFieldType(context.buildFullName(name), stored.getValue(), hasDocValues.getValue(), meta.getValue()), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, this ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index f11a3781fd19c..f2383148c31ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -123,14 +123,7 @@ public BooleanFieldMapper build(MapperBuilderContext context) { scriptValues(), meta.getValue() ); - return new BooleanFieldMapper( - name, - ft, - multiFieldsBuilder.build(this, context), - copyTo.build(), - context.isSourceSynthetic(), - this - ); + return new BooleanFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, context.isSourceSynthetic(), this); } private FieldValues scriptValues() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index ccf4953608f4d..7698c361f55d3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -205,7 +205,7 @@ public CompletionFieldMapper build(MapperBuilderContext context) { CompletionFieldType ft = new CompletionFieldType(context.buildFullName(name), completionAnalyzer, meta.getValue()); ft.setContextMappings(contexts.getValue()); - return new CompletionFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this); + return new CompletionFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, this); } private void checkCompletionContextsLimit() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 3bf593627faa9..9579d921c4176 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -358,7 +358,7 @@ public DateFieldMapper build(MapperBuilderContext context) { ); Long nullTimestamp = parseNullValue(ft); - return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); + return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, nullTimestamp, resolution, this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 8119cf5e299d0..a9d90f80c8a18 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -528,7 +528,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws */ public static class CopyTo { - private static final CopyTo EMPTY = new CopyTo(Collections.emptyList()); + private static final CopyTo EMPTY = new CopyTo(List.of()); public static CopyTo empty() { return EMPTY; @@ -537,7 +537,14 @@ public static CopyTo empty() { private final List copyToFields; private CopyTo(List copyToFields) { - this.copyToFields = copyToFields; + this.copyToFields = List.copyOf(copyToFields); + } + + public CopyTo withAddedFields(List fields) { + if (fields.isEmpty()) { + return this; + } + return new CopyTo(CollectionUtils.concatLists(copyToFields, fields)); } public XContentBuilder toXContent(XContentBuilder builder) throws IOException { @@ -551,31 +558,6 @@ public XContentBuilder toXContent(XContentBuilder builder) throws IOException { return builder; } - public static class Builder { - private final List copyToBuilders = new ArrayList<>(); - - public Builder add(String field) { - copyToBuilders.add(field); - return this; - } - - public boolean hasValues() { - return copyToBuilders.isEmpty() == false; - } - - public CopyTo build() { - if (copyToBuilders.isEmpty()) { - return EMPTY; - } - return new CopyTo(Collections.unmodifiableList(copyToBuilders)); - } - - public void reset(CopyTo copyTo) { - copyToBuilders.clear(); - copyToBuilders.addAll(copyTo.copyToFields); - } - } - public List copyToFields() { return copyToFields; } @@ -1216,7 +1198,7 @@ void check() { public abstract static class Builder extends Mapper.Builder implements ToXContentFragment { protected final MultiFields.Builder multiFieldsBuilder = new MultiFields.Builder(); - protected final CopyTo.Builder copyTo = new CopyTo.Builder(); + protected CopyTo copyTo = CopyTo.EMPTY; /** * Creates a new Builder with a field name @@ -1246,7 +1228,7 @@ protected void merge(FieldMapper in, Conflicts conflicts, MapperBuilderContext m for (FieldMapper newSubField : in.multiFields.mappers) { multiFieldsBuilder.update(newSubField, childContext); } - this.copyTo.reset(in.copyTo); + this.copyTo = in.copyTo; validate(); } @@ -1276,7 +1258,7 @@ protected void addScriptValidation( if (s != null && multiFieldsBuilder.hasMultiFields()) { throw new MapperParsingException("Cannot define multifields on a field with a script"); } - if (s != null && copyTo.hasValues()) { + if (s != null && copyTo.copyToFields().isEmpty() == false) { throw new MapperParsingException("Cannot define copy_to parameter on a field with a script"); } }); @@ -1324,7 +1306,7 @@ public final void parse(String name, MappingParserContext parserContext, Map { - TypeParsers.parseCopyFields(propNode).forEach(copyTo::add); + copyTo = copyTo.withAddedFields(TypeParsers.parseCopyFields(propNode)); iterator.remove(); continue; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 815148ea0598f..4f96723235035 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -213,7 +213,7 @@ public FieldMapper build(MapperBuilderContext context) { indexMode ); if (this.script.get() == null) { - return new GeoPointFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), geoParser, this); + return new GeoPointFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, geoParser, this); } return new GeoPointFieldMapper(name, ft, geoParser, this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 37ccdfe3da36c..7ae410a1a9dcb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -110,7 +110,7 @@ public GeoShapeFieldMapper build(MapperBuilderContext context) { name, ft, multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, new GeoShapeIndexer(orientation.get().value(), context.buildFullName(name)), geoShapeParser, this diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 259f51027ce73..49339c756bc7f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -178,7 +178,7 @@ public IpFieldMapper build(MapperBuilderContext context) { dimension.getValue() ), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, context.isSourceSynthetic(), this ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 31ca329d03375..e68e433353c83 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -323,7 +323,7 @@ public KeywordFieldMapper build(MapperBuilderContext context) { fieldtype, buildFieldType(context, fieldtype), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, context.isSourceSynthetic(), this ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 7590fd36d849a..34763eda69a28 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -268,14 +268,7 @@ protected Parameter[] getParameters() { @Override public NumberFieldMapper build(MapperBuilderContext context) { MappedFieldType ft = new NumberFieldType(context.buildFullName(name), this); - return new NumberFieldMapper( - name, - ft, - multiFieldsBuilder.build(this, context), - copyTo.build(), - context.isSourceSynthetic(), - this - ); + return new NumberFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, context.isSourceSynthetic(), this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java index a7de1194ddceb..4ad0873b66d50 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java @@ -91,13 +91,7 @@ protected Parameter[] getParameters() { @Override public PlaceHolderFieldMapper build(MapperBuilderContext context) { PlaceHolderFieldType mappedFieldType = new PlaceHolderFieldType(context.buildFullName(name), type, Map.of()); - return new PlaceHolderFieldMapper( - name, - mappedFieldType, - multiFieldsBuilder.build(this, context), - copyTo.build(), - unknownParams - ); + return new PlaceHolderFieldMapper(name, mappedFieldType, multiFieldsBuilder.build(this, context), copyTo, unknownParams); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index a7f0bedc7c060..fcd2a425a6625 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -163,7 +163,7 @@ protected RangeFieldType setupFieldType(MapperBuilderContext context) { @Override public RangeFieldMapper build(MapperBuilderContext context) { RangeFieldType ft = setupFieldType(context); - return new RangeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), type, this); + return new RangeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, type, this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 6d2c5b1cb71ac..384a35ca4156e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -469,7 +469,7 @@ public TextFieldMapper build(MapperBuilderContext context) { throw new MapperParsingException("Cannot use reserved field name [" + mapper.name() + "]"); } } - return new TextFieldMapper(name, fieldType, tft, prefixFieldInfo, phraseFieldInfo, multiFields, copyTo.build(), this); + return new TextFieldMapper(name, fieldType, tft, prefixFieldInfo, phraseFieldInfo, multiFields, copyTo, this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 35f2d37fa0dd6..51be6290df657 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -208,7 +208,6 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { if (multiFields.iterator().hasNext()) { throw new IllegalArgumentException(CONTENT_TYPE + " field [" + name + "] does not support [fields]"); } - CopyTo copyTo = this.copyTo.build(); if (copyTo.copyToFields().isEmpty() == false) { throw new IllegalArgumentException(CONTENT_TYPE + " field [" + name + "] does not support [copy_to]"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 155dac09f0b86..912e6ab5f442b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -194,7 +194,7 @@ public DenseVectorFieldMapper build(MapperBuilderContext context) { indexOptions.getValue(), indexVersionCreated, multiFieldsBuilder.build(this, context), - copyTo.build() + copyTo ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java index 082c2d898e637..758ad8e508ea3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java @@ -66,7 +66,7 @@ public SparseVectorFieldMapper build(MapperBuilderContext context) { name, new SparseVectorFieldType(context.buildFullName(name), meta.getValue()), multiFieldsBuilder.build(this, context), - copyTo.build() + copyTo ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 248d94cb67a63..8ff319134b964 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -174,7 +174,7 @@ protected Parameter[] getParameters() { @Override public FieldMapper build(MapperBuilderContext context) { - return new TestMapper(name(), context.buildFullName(name), multiFieldsBuilder.build(this, context), copyTo.build(), this); + return new TestMapper(name(), context.buildFullName(name), multiFieldsBuilder.build(this, context), copyTo, this); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 0034be7100df6..cb028c746a8cd 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -11,6 +11,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import java.util.Collections; +import java.util.List; // this sucks how much must be overridden just do get a dummy field mapper... public class MockFieldMapper extends FieldMapper { @@ -80,14 +81,14 @@ public Builder addMultiField(Builder builder) { } public Builder copyTo(String field) { - this.copyTo.add(field); + this.copyTo = copyTo.withAddedFields(List.of(field)); return this; } @Override public MockFieldMapper build(MapperBuilderContext context) { MultiFields multiFields = multiFieldsBuilder.build(this, context); - return new MockFieldMapper(name(), fieldType, multiFields, copyTo.build()); + return new MockFieldMapper(name(), fieldType, multiFields, copyTo); } } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 08969b7a779dc..08fad5dd3b83c 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -96,7 +96,7 @@ public HistogramFieldMapper build(MapperBuilderContext context) { name, new HistogramFieldType(context.buildFullName(name), meta.getValue()), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, this ); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 1c5e8d05ccf38..4e04f85383def 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -203,7 +203,7 @@ public UnsignedLongFieldMapper build(MapperBuilderContext context) { metric.getValue(), indexMode ); - return new UnsignedLongFieldMapper(name, fieldType, multiFieldsBuilder.build(this, context), copyTo.build(), this); + return new UnsignedLongFieldMapper(name, fieldType, multiFieldsBuilder.build(this, context), copyTo, this); } } diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index f0d86da6efe8d..e6dc15550f418 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -116,7 +116,7 @@ public VersionStringFieldMapper build(MapperBuilderContext context) { fieldtype, buildFieldType(context, fieldtype), multiFieldsBuilder.build(this, context), - copyTo.build() + copyTo ); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 1cba21a526552..55929a1c1b83e 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -171,7 +171,7 @@ public GeoShapeWithDocValuesFieldMapper build(MapperBuilderContext context) { name, ft, multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, new GeoShapeIndexer(orientation.get().value(), ft.name()), parser, this diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java index bf63f2d84c189..378b78111ab19 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java @@ -119,7 +119,7 @@ public FieldMapper build(MapperBuilderContext context) { parser, meta.get() ); - return new PointFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), parser, this); + return new PointFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index 1b784595caa16..127a4fd1050cd 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -122,7 +122,7 @@ public ShapeFieldMapper build(MapperBuilderContext context) { parser, meta.get() ); - return new ShapeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), parser, this); + return new ShapeFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); } } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 469e140eb6b76..e336d1031b292 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -242,7 +242,7 @@ public WildcardFieldMapper build(MapperBuilderContext context) { ignoreAbove.get(), context.isSourceSynthetic(), multiFieldsBuilder.build(this, context), - copyTo.build(), + copyTo, nullValue.get(), indexVersionCreated ); From 7433b5122c2d092fea60d802c8a7ed293e6a6bea Mon Sep 17 00:00:00 2001 From: Dianna Hohensee Date: Fri, 8 Sep 2023 13:40:21 -0400 Subject: [PATCH 49/69] Revert "Call TransportService#registerRequestHandler with Executor (#98998)" This reverts commit df38bc8a84a7a47d98d16680f898596edb039122. --- .../common/GrokProcessorGetActionTests.java | 7 +- .../TransportRankEvalActionTests.java | 4 +- .../reindex/UpdateByQueryWithScriptTests.java | 5 - ...rossClusterSearchUnavailableClusterIT.java | 54 +-- .../flush/TransportShardFlushAction.java | 2 +- .../TransportFieldCapabilitiesAction.java | 2 +- .../action/search/SearchTransportService.java | 25 +- .../TransportOpenPointInTimeAction.java | 3 +- .../support/HandledTransportAction.java | 9 +- .../broadcast/TransportBroadcastAction.java | 2 +- .../node/TransportBroadcastByNodeAction.java | 2 +- .../TransportBroadcastUnpromotableAction.java | 7 +- .../support/nodes/TransportNodesAction.java | 2 +- .../TransportReplicationAction.java | 12 +- ...ransportInstanceSingleOperationAction.java | 4 +- .../shard/TransportSingleShardAction.java | 10 +- .../support/tasks/TransportTasksAction.java | 7 +- .../action/shard/ShardStateAction.java | 5 +- .../cluster/coordination/Coordinator.java | 2 +- .../coordination/FollowersChecker.java | 2 +- .../cluster/coordination/JoinHelper.java | 7 +- .../coordination/JoinValidationService.java | 2 +- .../cluster/coordination/LeaderChecker.java | 2 +- .../PublicationTransportHandler.java | 2 +- .../StatefulPreVoteCollector.java | 2 +- .../elasticsearch/discovery/PeerFinder.java | 2 +- .../gateway/LocalAllocateDangledIndices.java | 3 +- .../recovery/PeerRecoverySourceService.java | 4 +- .../recovery/PeerRecoveryTargetService.java | 16 +- .../indices/store/IndicesStore.java | 3 +- .../VerifyNodeRepositoryAction.java | 2 +- .../tasks/TaskCancellationService.java | 5 +- .../transport/RemoteClusterService.java | 3 +- .../transport/TransportActionProxy.java | 3 +- .../transport/TransportService.java | 40 ++ ...nsportDeleteDesiredBalanceActionTests.java | 16 +- ...TransportGetDesiredBalanceActionTests.java | 39 +- .../ClusterFormationInfoActionTests.java | 6 - .../MasterHistoryActionTests.java | 6 - ...ransportUpdateDesiredNodesActionTests.java | 20 +- .../ReservedRepositoryActionTests.java | 7 +- .../settings/ClusterGetSettingsTests.java | 7 +- .../ClusterUpdateSettingsRequestTests.java | 7 +- .../TransportCreateIndexActionTests.java | 8 +- .../TransportRolloverActionTests.java | 6 - .../TransportUpdateSettingsActionTests.java | 8 +- ...vedComposableIndexTemplateActionTests.java | 25 +- ...ActionIndicesThatCannotBeCreatedTests.java | 10 +- .../bulk/TransportBulkActionIngestTests.java | 5 - ...StateAwareHandledTransportActionTests.java | 6 +- .../replication/PostWriteRefreshTests.java | 4 +- .../TransportWriteActionTests.java | 2 +- .../PeerRecoverySourceServiceTests.java | 4 +- .../tasks/BanFailureLoggingTests.java | 2 +- .../RemoteClusterConnectionTests.java | 78 ++-- .../SniffConnectionStrategyTests.java | 5 +- .../transport/TransportActionProxyTests.java | 99 ++--- ...ortServiceDeserializationFailureTests.java | 3 +- .../TransportServiceLifecycleTests.java | 2 +- .../org/elasticsearch/test/MockUtils.java | 71 --- .../AbstractSimpleTransportTestCase.java | 410 +++++++----------- .../DisruptableMockTransportTests.java | 7 +- .../AnalyticsInfoTransportActionTests.java | 17 +- .../TransportAnalyticsStatsActionTests.java | 6 - ...ortDeleteAutoscalingPolicyActionTests.java | 13 +- ...nsportGetAutoscalingPolicyActionTests.java | 13 +- ...nsportPutAutoscalingPolicyActionTests.java | 13 +- .../ccr/CCRInfoTransportActionTests.java | 18 +- .../ClearCcrRestoreSessionActionTests.java | 3 - .../GetCcrRestoreFileChunkActionTests.java | 3 - .../action/TransportTermsEnumAction.java | 2 +- .../action/TransportXPackInfoActionTests.java | 4 +- ...nsportNodeDeprecationCheckActionTests.java | 7 - .../InternalExecutePolicyActionTests.java | 4 +- .../eql/EqlInfoTransportActionTests.java | 13 +- .../compute/operator/DriverTaskRunner.java | 4 +- .../operator/exchange/ExchangeService.java | 4 +- .../esql/enrich/EnrichLookupService.java | 7 +- .../xpack/esql/plugin/ComputeService.java | 9 +- .../esql/EsqlInfoTransportActionTests.java | 12 +- .../graph/GraphInfoTransportActionTests.java | 12 +- ...portPutSamlServiceProviderActionTests.java | 5 +- ...ndexLifecycleInfoTransportActionTests.java | 23 +- .../TransportDeleteLifecycleActionTests.java | 7 +- .../TransportPutLifecycleActionTests.java | 7 +- .../action/TransportStopILMActionTests.java | 7 +- .../TransportDeletePipelineActionTests.java | 4 +- .../TransportGetPipelineActionTests.java | 14 +- ...chineLearningInfoTransportActionTests.java | 14 +- .../action/TransportCloseJobActionTests.java | 7 +- ...TransportDeleteExpiredDataActionTests.java | 5 - ...nsportFinalizeJobExecutionActionTests.java | 6 +- .../MonitoringInfoTransportActionTests.java | 19 +- .../TransportMonitoringBulkActionTests.java | 5 - .../action/TransportRollupSearchAction.java | 4 +- .../RollupInfoTransportActionTests.java | 17 +- ...CrossClusterAccessHeadersForCcsRestIT.java | 60 ++- .../SecurityInfoTransportActionTests.java | 23 +- ...rtCreateCrossClusterApiKeyActionTests.java | 9 +- .../TransportGrantApiKeyActionTests.java | 8 +- ...rtUpdateCrossClusterApiKeyActionTests.java | 7 +- ...nsportProfileHasPrivilegesActionTests.java | 5 - .../TransportClearRealmCacheActionTests.java | 8 +- ...tCreateServiceAccountTokenActionTests.java | 5 +- ...tDeleteServiceAccountTokenActionTests.java | 4 +- ...TransportGetServiceAccountActionTests.java | 4 +- ...tServiceAccountCredentialsActionTests.java | 5 +- .../TransportCreateTokenActionTests.java | 16 +- .../TransportInvalidateTokenActionTests.java | 10 +- .../TransportHasPrivilegesActionTests.java | 9 +- ...ty4ServerTransportAuthenticationTests.java | 3 +- ...ransportDeleteShutdownNodeActionTests.java | 4 +- .../TransportPutShutdownNodeActionTests.java | 4 +- ...vedSnapshotLifecycleStateServiceTests.java | 13 +- .../action/TransportStopSLMActionTests.java | 8 +- .../SpatialInfoTransportActionTests.java | 17 +- .../SpatialStatsTransportActionTests.java | 3 +- .../sql/SqlInfoTransportActionTests.java | 14 +- .../TransformInfoTransportActionTests.java | 7 +- .../WatcherInfoTransportActionTests.java | 14 +- .../actions/TransportAckWatchActionTests.java | 3 +- .../actions/TransportPutWatchActionTests.java | 3 +- .../TransportWatcherStatsActionTests.java | 3 +- 123 files changed, 628 insertions(+), 1074 deletions(-) delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/MockUtils.java diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java index 09801eeda6437..c2110a14db7a6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.grok.PatternBank; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -61,9 +60,8 @@ public void testResponseSerialization() throws Exception { public void testResponseSorting() { List sortedKeys = new ArrayList<>(LEGACY_TEST_PATTERNS.bank().keySet()); Collections.sort(sortedKeys); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GrokProcessorGetAction.TransportAction transportAction = new GrokProcessorGetAction.TransportAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), LEGACY_TEST_PATTERNS, ECS_TEST_PATTERNS @@ -111,9 +109,8 @@ public void onFailure(Exception e) { public void testEcsCompatibilityMode() { List sortedKeys = new ArrayList<>(ECS_TEST_PATTERNS.bank().keySet()); Collections.sort(sortedKeys); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GrokProcessorGetAction.TransportAction transportAction = new GrokProcessorGetAction.TransportAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), LEGACY_TEST_PATTERNS, ECS_TEST_PATTERNS diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java index 3838916c15c64..1b12f45ac8b51 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -75,11 +74,10 @@ public void multiSearch(MultiSearchRequest request, ActionListener { channel.sendResponse(new SearchShardsResponse(List.of(), List.of(), Collections.emptyMap())); } ); - newService.registerRequestHandler( - SearchAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - SearchRequest::new, - (request, channel, task) -> { - InternalSearchResponse response = new InternalSearchResponse( - new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - null, - false, - null, - 1 - ); - SearchResponse searchResponse = new SearchResponse( - response, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); - channel.sendResponse(searchResponse); - } - ); + newService.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { + InternalSearchResponse response = new InternalSearchResponse( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), + InternalAggregations.EMPTY, + null, + null, + false, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + response, + null, + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + channel.sendResponse(searchResponse); + }); newService.registerRequestHandler( ClusterStateAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java index fbcfe415039ff..0bd4458056c20 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java @@ -62,7 +62,7 @@ public TransportShardFlushAction( ); transportService.registerRequestHandler( PRE_SYNCED_FLUSH_ACTION_NAME, - threadPool.executor(ThreadPool.Names.FLUSH), + ThreadPool.Names.FLUSH, PreShardSyncedFlushRequest::new, new PreSyncedFlushTransportHandler(indicesService) ); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 1dde340e0f527..79b9fa5099467 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -92,7 +92,7 @@ public TransportFieldCapabilitiesAction( this.indicesService = indicesService; transportService.registerRequestHandler( ACTION_NODE_NAME, - this.searchCoordinationExecutor, + ThreadPool.Names.SEARCH_COORDINATION, FieldCapabilitiesNodeRequest::new, new NodeTransportHandler() ); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java index d02958567a873..9714fc7574418 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.CountDown; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.search.CanMatchShardResponse; import org.elasticsearch.search.SearchPhaseResult; @@ -451,7 +450,7 @@ public void writeTo(StreamOutput out) throws IOException { public static void registerRequestHandler(TransportService transportService, SearchService searchService) { transportService.registerRequestHandler( FREE_CONTEXT_SCROLL_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, ScrollFreeContextRequest::new, (request, channel, task) -> { boolean freed = searchService.freeReaderContext(request.id()); @@ -461,7 +460,7 @@ public static void registerRequestHandler(TransportService transportService, Sea TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_SCROLL_ACTION_NAME, false, SearchFreeContextResponse::new); transportService.registerRequestHandler( FREE_CONTEXT_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, SearchFreeContextRequest::new, (request, channel, task) -> { boolean freed = searchService.freeReaderContext(request.id()); @@ -471,7 +470,7 @@ public static void registerRequestHandler(TransportService transportService, Sea TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, false, SearchFreeContextResponse::new); transportService.registerRequestHandler( CLEAR_SCROLL_CONTEXTS_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, TransportRequest.Empty::new, (request, channel, task) -> { searchService.freeAllScrollContexts(); @@ -487,7 +486,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( DFS_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, ShardSearchRequest::new, (request, channel, task) -> searchService.executeDfsPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)) ); @@ -496,7 +495,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, ShardSearchRequest::new, (request, channel, task) -> searchService.executeQueryPhase( request, @@ -513,7 +512,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_ID_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, QuerySearchRequest::new, (request, channel, task) -> { searchService.executeQueryPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -523,7 +522,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_SCROLL_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, InternalScrollSearchRequest::new, (request, channel, task) -> { searchService.executeQueryPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -533,7 +532,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_FETCH_SCROLL_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, InternalScrollSearchRequest::new, (request, channel, task) -> { searchService.executeFetchPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -543,7 +542,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( FETCH_ID_SCROLL_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, ShardFetchRequest::new, (request, channel, task) -> { searchService.executeFetchPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -553,7 +552,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( FETCH_ID_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, true, true, ShardFetchSearchRequest::new, @@ -566,7 +565,7 @@ public static void registerRequestHandler(TransportService transportService, Sea // this is cheap, it does not fetch during the rewrite phase, so we can let it quickly execute on a networking thread transportService.registerRequestHandler( QUERY_CAN_MATCH_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, ShardSearchRequest::new, (request, channel, task) -> { searchService.canMatch(request, new ChannelActionListener<>(channel)); @@ -576,7 +575,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_CAN_MATCH_NODE_NAME, - transportService.getThreadPool().executor(ThreadPool.Names.SEARCH_COORDINATION), + ThreadPool.Names.SEARCH_COORDINATION, CanMatchNodeRequest::new, (request, channel, task) -> { searchService.canMatch(request, new ChannelActionListener<>(channel)); diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java index eb854be99562e..9c78e5ad62aea 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchPhaseResult; @@ -69,7 +68,7 @@ public TransportOpenPointInTimeAction( this.searchTransportService = searchTransportService; transportService.registerRequestHandler( OPEN_SHARD_READER_CONTEXT_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, ShardOpenReaderRequest::new, new ShardOpenReaderRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java index a0e36a538913d..24668d7130d52 100644 --- a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java @@ -61,14 +61,7 @@ protected HandledTransportAction( String executor ) { super(actionName, actionFilters, transportService.getTaskManager()); - transportService.registerRequestHandler( - actionName, - transportService.getThreadPool().executor(executor), - false, - canTripCircuitBreaker, - requestReader, - new TransportHandler() - ); + transportService.registerRequestHandler(actionName, executor, false, canTripCircuitBreaker, requestReader, new TransportHandler()); } class TransportHandler implements TransportRequestHandler { diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java index ddd2cc43005e2..570514355da56 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java @@ -75,7 +75,7 @@ protected TransportBroadcastAction( transportService.registerRequestHandler( transportShardAction, - this.executor, + executor, shardRequestReader, (request, channel, task) -> ActionListener.completeWith( new ChannelActionListener<>(channel), diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index 4f78d68daa8f2..15c28bdf33c87 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -119,7 +119,7 @@ public TransportBroadcastByNodeAction( transportService.registerRequestHandler( transportNodeBroadcastAction, - this.executor, + executor, false, canTripCircuitBreaker, NodeRequest::new, diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java index 579ed101b56ae..d5c7fc763fba6 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java @@ -60,12 +60,7 @@ protected TransportBroadcastUnpromotableAction( this.transportUnpromotableAction = actionName + "[u]"; this.executor = transportService.getThreadPool().executor(executor); - transportService.registerRequestHandler( - transportUnpromotableAction, - this.executor, - requestReader, - new UnpromotableTransportHandler() - ); + transportService.registerRequestHandler(transportUnpromotableAction, executor, requestReader, new UnpromotableTransportHandler()); } protected abstract void unpromotableShardOperation(Task task, Request request, ActionListener listener); diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java index b45c448ecc52c..1efe70bcb5adc 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java @@ -82,7 +82,7 @@ protected TransportNodesAction( this.transportService = Objects.requireNonNull(transportService); this.finalExecutor = threadPool.executor(executor); this.transportNodeAction = actionName + "[n]"; - transportService.registerRequestHandler(transportNodeAction, finalExecutor, nodeRequest, new NodeTransportHandler()); + transportService.registerRequestHandler(transportNodeAction, executor, nodeRequest, new NodeTransportHandler()); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index d1e2b6dd98faa..5651af11719cf 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -38,7 +38,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; @@ -188,16 +187,11 @@ protected TransportReplicationAction( this.retryTimeout = REPLICATION_RETRY_TIMEOUT.get(settings); this.forceExecutionOnPrimary = forceExecutionOnPrimary; - transportService.registerRequestHandler( - actionName, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - requestReader, - this::handleOperationRequest - ); + transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, requestReader, this::handleOperationRequest); transportService.registerRequestHandler( transportPrimaryAction, - threadPool.executor(executor), + executor, forceExecutionOnPrimary, true, in -> new ConcreteShardRequest<>(requestReader, in), @@ -207,7 +201,7 @@ protected TransportReplicationAction( // we must never reject on because of thread pool capacity on replicas transportService.registerRequestHandler( transportReplicaAction, - threadPool.executor(executor), + executor, true, true, in -> new ConcreteReplicaRequest<>(replicaRequestReader, in), diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index c526a2679306d..63ff4cef5a924 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; @@ -34,6 +33,7 @@ import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.threadpool.ThreadPool.Names; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; @@ -72,7 +72,7 @@ protected TransportInstanceSingleOperationAction( this.transportService = transportService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.shardActionName = actionName + "[s]"; - transportService.registerRequestHandler(shardActionName, EsExecutors.DIRECT_EXECUTOR_SERVICE, request, new ShardTransportHandler()); + transportService.registerRequestHandler(shardActionName, Names.SAME, request, new ShardTransportHandler()); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java index 9d980fabc6714..001a99ecd505d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.LoggerMessageFormat; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; @@ -80,14 +79,9 @@ protected TransportSingleShardAction( this.executor = executor; if (isSubAction() == false) { - transportService.registerRequestHandler(actionName, EsExecutors.DIRECT_EXECUTOR_SERVICE, request, new TransportHandler()); + transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, request, new TransportHandler()); } - transportService.registerRequestHandler( - transportShardAction, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - request, - new ShardTransportHandler() - ); + transportService.registerRequestHandler(transportShardAction, ThreadPool.Names.SAME, request, new ShardTransportHandler()); } /** diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index fb29e8a838db6..b509008af2b4f 100644 --- a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -79,12 +79,7 @@ protected TransportTasksAction( this.responsesReader = responsesReader; this.responseReader = responseReader; - transportService.registerRequestHandler( - transportNodeAction, - transportService.getThreadPool().executor(nodeExecutor), - NodeTaskRequest::new, - new NodeTransportHandler() - ); + transportService.registerRequestHandler(transportNodeAction, nodeExecutor, NodeTaskRequest::new, new NodeTransportHandler()); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index f9aa2c6c234be..97bf6cc01c4e5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -36,7 +36,6 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; @@ -99,13 +98,13 @@ public ShardStateAction( transportService.registerRequestHandler( SHARD_STARTED_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, StartedShardEntry::new, new ShardStartedTransportHandler(clusterService, new ShardStartedClusterStateTaskExecutor(allocationService, rerouteService)) ); transportService.registerRequestHandler( SHARD_FAILED_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, FailedShardEntry::new, new ShardFailedTransportHandler(clusterService, new ShardFailedClusterStateTaskExecutor(allocationService, rerouteService)) ); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 7ccea8e99918b..dc52791a5d5e2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -271,7 +271,7 @@ public Coordinator( ); transportService.registerRequestHandler( COMMIT_STATE_ACTION_NAME, - this.clusterCoordinationExecutor, + Names.CLUSTER_COORDINATION, false, false, ApplyCommitRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java index 99ab8650a52d8..ad2faaccf0e96 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java @@ -124,7 +124,7 @@ public FollowersChecker( updateFastResponseState(0, Mode.CANDIDATE); transportService.registerRequestHandler( FOLLOWER_CHECK_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + Names.SAME, false, false, FollowerCheckRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java index 247034c88ed62..ce2754fa3854c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; @@ -115,7 +114,7 @@ public class JoinHelper { transportService.registerRequestHandler( JOIN_ACTION_NAME, - transportService.getThreadPool().executor(Names.CLUSTER_COORDINATION), + Names.CLUSTER_COORDINATION, false, false, JoinRequest::new, @@ -127,7 +126,7 @@ public class JoinHelper { transportService.registerRequestHandler( START_JOIN_ACTION_NAME, - transportService.getThreadPool().executor(Names.CLUSTER_COORDINATION), + Names.CLUSTER_COORDINATION, false, false, StartJoinRequest::new, @@ -140,7 +139,7 @@ public class JoinHelper { transportService.registerRequestHandler( JOIN_PING_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, false, false, TransportRequest.Empty::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java index d9911ad12df84..248137f03fcaa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java @@ -121,7 +121,7 @@ public JoinValidationService( final var dataPaths = Environment.PATH_DATA_SETTING.get(settings); transportService.registerRequestHandler( JoinValidationService.JOIN_VALIDATE_ACTION_NAME, - this.responseExecutor, + ThreadPool.Names.CLUSTER_COORDINATION, ValidateJoinRequest::new, (request, channel, task) -> { final var remoteState = request.getOrReadState(); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java index 9fcae5bcf67f8..8a20e8e56d751 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java @@ -113,7 +113,7 @@ public class LeaderChecker { transportService.registerRequestHandler( LEADER_CHECK_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + Names.SAME, false, false, LeaderCheckRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java index 781a05d535b16..45079b2bccd60 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java @@ -106,7 +106,7 @@ public PublicationTransportHandler( transportService.registerRequestHandler( PUBLISH_STATE_ACTION_NAME, - this.clusterCoordinationExecutor, + ThreadPool.Names.CLUSTER_COORDINATION, false, false, BytesTransportRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java index 7bc3514206ffe..28ab04e5a7ccc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java @@ -64,7 +64,7 @@ public StatefulPreVoteCollector( transportService.registerRequestHandler( REQUEST_PRE_VOTE_ACTION_NAME, - this.clusterCoordinationExecutor, + Names.CLUSTER_COORDINATION, false, false, PreVoteRequest::new, diff --git a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java index 46e8f5210e447..046f0a1c64bb5 100644 --- a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java +++ b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java @@ -110,7 +110,7 @@ public PeerFinder( transportService.registerRequestHandler( REQUEST_PEERS_ACTION_NAME, - this.clusterCoordinationExecutor, + Names.CLUSTER_COORDINATION, false, false, PeersRequest::new, diff --git a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java index 0262e37dd74a9..d0579342f5bd0 100644 --- a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java +++ b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java @@ -35,6 +35,7 @@ import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestHandler; @@ -72,7 +73,7 @@ public LocalAllocateDangledIndices( this.indexMetadataVerifier = indexMetadataVerifier; transportService.registerRequestHandler( ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, AllocateDangledRequest::new, new AllocateDangledRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java index eac119f920f6a..21b8ca31f01a7 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java @@ -79,7 +79,7 @@ public PeerRecoverySourceService( // node. Upon receiving START_RECOVERY, the source node will initiate the peer recovery. transportService.registerRequestHandler( Actions.START_RECOVERY, - transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, StartRecoveryRequest::new, (request, channel, task) -> recover(request, task, new ChannelActionListener<>(channel)) ); @@ -89,7 +89,7 @@ public PeerRecoverySourceService( // action will fail and the target node will send a new START_RECOVERY request. transportService.registerRequestHandler( Actions.REESTABLISH_RECOVERY, - transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, ReestablishRecoveryRequest::new, (request, channel, task) -> reestablish(request, new ChannelActionListener<>(channel)) ); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java index b3b23ac14d158..5f06519f84c0c 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java @@ -122,7 +122,7 @@ public PeerRecoveryTargetService( transportService.registerRequestHandler( Actions.FILES_INFO, - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, RecoveryFilesInfoRequest::new, new RecoveryRequestHandler<>() { @Override @@ -140,7 +140,7 @@ protected void handleRequest(RecoveryFilesInfoRequest request, RecoveryTarget ta ); transportService.registerRequestHandler( Actions.RESTORE_FILE_FROM_SNAPSHOT, - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, RecoverySnapshotFileRequest::new, new RecoveryRequestHandler<>() { @Override @@ -151,13 +151,13 @@ protected void handleRequest(RecoverySnapshotFileRequest request, RecoveryTarget ); transportService.registerRequestHandler( Actions.FILE_CHUNK, - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, RecoveryFileChunkRequest::new, new FileChunkTransportRequestHandler() ); transportService.registerRequestHandler( Actions.CLEAN_FILES, - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, RecoveryCleanFilesRequest::new, new RecoveryRequestHandler<>() { @Override @@ -179,7 +179,7 @@ protected void handleRequest(RecoveryCleanFilesRequest request, RecoveryTarget t ); transportService.registerRequestHandler( Actions.PREPARE_TRANSLOG, - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, RecoveryPrepareForTranslogOperationsRequest::new, new RecoveryRequestHandler<>() { @Override @@ -194,13 +194,13 @@ protected void handleRequest( ); transportService.registerRequestHandler( Actions.TRANSLOG_OPS, - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, RecoveryTranslogOperationsRequest::new, new TranslogOperationsRequestHandler() ); transportService.registerRequestHandler( Actions.FINALIZE, - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, RecoveryFinalizeRecoveryRequest::new, new RecoveryRequestHandler<>() { @Override @@ -215,7 +215,7 @@ protected void handleRequest( ); transportService.registerRequestHandler( Actions.HANDOFF_PRIMARY_CONTEXT, - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, RecoveryHandoffPrimaryContextRequest::new, new HandoffPrimaryContextRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index f949dd59e8968..e12a38e383397 100644 --- a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -103,7 +102,7 @@ public IndicesStore( this.threadPool = threadPool; transportService.registerRequestHandler( ACTION_SHARD_EXISTS, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, ShardActiveRequest::new, new ShardActiveRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java index d940415c38916..ad6cbbaad1cdc 100644 --- a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java @@ -56,7 +56,7 @@ public VerifyNodeRepositoryAction( this.repositoriesService = repositoriesService; transportService.registerRequestHandler( ACTION_NAME, - transportService.getThreadPool().executor(ThreadPool.Names.SNAPSHOT), + ThreadPool.Names.SNAPSHOT, VerifyNodeRepositoryRequest::new, new VerifyNodeRepositoryRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java index 6ab95072727c0..39a76f3508daa 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java @@ -23,7 +23,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.NodeDisconnectedException; @@ -61,13 +60,13 @@ public TaskCancellationService(TransportService transportService) { this.deduplicator = new ResultDeduplicator<>(transportService.getThreadPool().getThreadContext()); transportService.registerRequestHandler( BAN_PARENT_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, BanParentTaskRequest::new, new BanParentRequestHandler() ); transportService.registerRequestHandler( CANCEL_CHILD_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, CancelChildRequest::new, new CancelChildRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index c38f4b26c665f..9542d4b366ded 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.CountDown; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.node.ReportingService; @@ -540,7 +539,7 @@ Collection getConnections() { static void registerRemoteClusterHandshakeRequestHandler(TransportService transportService) { transportService.registerRequestHandler( REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, false, false, TransportService.HandshakeRequest::new, diff --git a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java index aa28f8a76b58e..ed3432f66087b 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -189,7 +188,7 @@ public static void registerProxyActionWithDynamicResponseType( RequestHandlerRegistry requestHandler = service.getRequestHandler(action); service.registerRequestHandler( getProxyAction(action), - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, true, false, in -> cancellable diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 5369b9a9eec13..83fc21396e0f6 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -1118,6 +1118,46 @@ public static boolean isValidActionName(String actionName) { return false; } + /** + * Temporary passthrough function that continues to take a String rather than Executor type. + * + * @param action + * @param executor + * @param requestReader + * @param handler + * @param + */ + public void registerRequestHandler( + String action, + String executor, + Writeable.Reader requestReader, + TransportRequestHandler handler + ) { + registerRequestHandler(action, threadPool.executor(executor), requestReader, handler); + } + + /** + * Temporary passthrough function that continues to take a String rather than Executor type. + * + * @param action + * @param executor + * @param forceExecution + * @param canTripCircuitBreaker + * @param requestReader + * @param handler + * @param + */ + public void registerRequestHandler( + String action, + String executor, + boolean forceExecution, + boolean canTripCircuitBreaker, + Writeable.Reader requestReader, + TransportRequestHandler handler + ) { + registerRequestHandler(action, threadPool.executor(executor), forceExecution, canTripCircuitBreaker, requestReader, handler); + } + /** * Registers a new request handler * diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java index a2bfa68fdf25f..f5e812e7d0983 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ClusterServiceUtils; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.gateway.TestGatewayAllocator; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -57,20 +56,17 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class TransportDeleteDesiredBalanceActionTests extends ESAllocationTestCase { public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception { - var listener = new PlainActionFuture(); - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + var listener = new PlainActionFuture(); new TransportDeleteDesiredBalanceAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(AllocationService.class), @@ -135,12 +131,8 @@ public DesiredBalance compute( var listener = new PlainActionFuture(); - // TODO: temporary, remove in #97879 - TransportService transportService = mock(TransportService.class); - when(transportService.getThreadPool()).thenReturn(threadPool); - var action = new TransportDeleteDesiredBalanceAction( - transportService, + mock(TransportService.class), clusterService, threadPool, mock(ActionFilters.class), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java index 80f0b435645e6..804dbf66ac0d0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; @@ -46,7 +45,6 @@ import org.elasticsearch.test.AbstractChunkedSerializingTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.junit.Before; import java.util.HashMap; import java.util.List; @@ -61,7 +59,6 @@ import static org.elasticsearch.cluster.ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -69,27 +66,16 @@ public class TransportGetDesiredBalanceActionTests extends ESAllocationTestCase private final DesiredBalanceShardsAllocator desiredBalanceShardsAllocator = mock(DesiredBalanceShardsAllocator.class); private final ClusterInfoService clusterInfoService = mock(ClusterInfoService.class); - private TransportService transportService = mock(TransportService.class); - private ThreadPool threadPool = mock(ThreadPool.class); - private TransportGetDesiredBalanceAction transportGetDesiredBalanceAction; - - @Before - public void initialize() { - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - - transportGetDesiredBalanceAction = new TransportGetDesiredBalanceAction( - transportService, - mock(ClusterService.class), - threadPool, - mock(ActionFilters.class), - mock(IndexNameExpressionResolver.class), - desiredBalanceShardsAllocator, - clusterInfoService, - TEST_WRITE_LOAD_FORECASTER - ); - } + private final TransportGetDesiredBalanceAction transportGetDesiredBalanceAction = new TransportGetDesiredBalanceAction( + mock(TransportService.class), + mock(ClusterService.class), + mock(ThreadPool.class), + mock(ActionFilters.class), + mock(IndexNameExpressionResolver.class), + desiredBalanceShardsAllocator, + clusterInfoService, + TEST_WRITE_LOAD_FORECASTER + ); private static DesiredBalanceResponse execute(TransportGetDesiredBalanceAction action, ClusterState clusterState) throws Exception { return PlainActionFuture.get( @@ -110,10 +96,11 @@ private DesiredBalanceResponse executeAction(ClusterState clusterState) throws E public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception { var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataWithConfiguredAllocator(BALANCED_ALLOCATOR)).build(); + final var action = new TransportGetDesiredBalanceAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(ShardsAllocator.class), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java index f190c422e165c..e560959dfacf8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.monitor.StatusInfo; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; @@ -31,7 +30,6 @@ import java.util.List; import java.util.Map; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -164,10 +162,6 @@ public void testTransportDoExecute() { Coordinator coordinator = mock(Coordinator.class); ClusterFormationFailureHelper.ClusterFormationState clusterFormationState = getClusterFormationState(); when(coordinator.getClusterFormationState()).thenReturn(clusterFormationState); - - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); ClusterFormationInfoAction.TransportAction action = new ClusterFormationInfoAction.TransportAction( transportService, actionFilters, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java index f362327a3d8e4..3674a711d8968 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; import org.elasticsearch.threadpool.ThreadPool; @@ -25,7 +24,6 @@ import java.util.ArrayList; import java.util.List; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -85,10 +83,6 @@ public void testTransportDoExecute() { when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis()); MasterHistory masterHistory = new MasterHistory(threadPool, clusterService); when(masterHistoryService.getLocalMasterHistory()).thenReturn(masterHistory); - - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); MasterHistoryAction.TransportAction action = new MasterHistoryAction.TransportAction( transportService, actionFilters, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java index 508cb9b304c59..e4e8293810f56 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.tasks.Task; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -55,12 +54,10 @@ public void validate(List desiredNodes) {} }; public void testWriteBlocks() { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_OP_SETTINGS_VALIDATOR, @@ -82,12 +79,10 @@ public void testWriteBlocks() { } public void testNoBlocks() { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_OP_SETTINGS_VALIDATOR, @@ -108,13 +103,10 @@ public void validate(List desiredNodes) { } }; ClusterService clusterService = mock(ClusterService.class); - - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - transportService, + mock(TransportService.class), clusterService, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), validator, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java index 1b3e0ae6fe7bf..3b3a996e50cd3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentParser; @@ -133,16 +132,14 @@ public Repository create(RepositoryMetadata metadata) { } }; - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); RepositoriesService repositoriesService = spy( new RepositoriesService( Settings.EMPTY, mock(ClusterService.class), - transportService, + mock(TransportService.class), Map.of(), Map.of("fs", fsFactory), - threadPool, + mock(ThreadPool.class), null ) ); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java index 00b500254883c..9e757894bcdda 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -48,12 +47,10 @@ public void testRequestConstruction() { public void testTransportFilters() throws Exception { final SettingsFilter filter = new SettingsFilter(List.of("persistent.foo.filtered", "transient.foo.filtered")); - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportClusterGetSettingsAction action = new TransportClusterGetSettingsAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), filter, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java index 6373b94ffb94a..ebe77f4d3ed4c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -86,12 +85,10 @@ private static ClusterUpdateSettingsRequest createTestItem() { public void testOperatorHandler() throws IOException { ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - final ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportClusterUpdateSettingsAction action = new TransportClusterUpdateSettingsAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), clusterSettings diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java index d68641d04dd74..671bf13fb0e77 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -91,13 +90,10 @@ public void setUp() throws Exception { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, SYSTEM_INDICES); this.metadataCreateIndexService = mock(MetadataCreateIndexService.class); - - final ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); this.action = new TransportCreateIndexAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), metadataCreateIndexService, mock(ActionFilters.class), indexNameExpressionResolver, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index bb1cf85013498..12308036689c6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -37,7 +37,6 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.cache.query.QueryCacheStats; @@ -79,7 +78,6 @@ import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -356,10 +354,6 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr EmptySystemIndices.INSTANCE, WriteLoadForecaster.DEFAULT ); - - // TODO: temporary, remove in #97879 - when(mockTransportService.getThreadPool()).thenReturn(mockThreadPool); - when(mockThreadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); final TransportRolloverAction transportRolloverAction = new TransportRolloverAction( mockTransportService, mockClusterService, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java index 807462b6052c3..0e1581606f803 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -69,13 +68,10 @@ public void setUp() throws Exception { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, SYSTEM_INDICES); MetadataUpdateSettingsService metadataUpdateSettingsService = mock(MetadataUpdateSettingsService.class); - - final ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); this.action = new TransportUpdateSettingsAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), metadataUpdateSettingsService, mock(ActionFilters.class), indexNameExpressionResolver, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java index 28476a0d8b839..553dd2368e5bc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java @@ -40,7 +40,6 @@ import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -674,12 +673,10 @@ public void testAddRemoveIndexTemplatesWithOverlap() throws Exception { } public void testHandlerCorrectness() { - final ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putIndexAction = new TransportPutComposableIndexTemplateAction( - transportService, + mock(TransportService.class), null, - threadPool, + mock(ThreadPool.class), null, mock(ActionFilters.class), null @@ -690,9 +687,9 @@ public void testHandlerCorrectness() { containsInAnyOrder(reservedComposableIndexName("aaa")) ); var delIndexAction = new TransportDeleteComposableIndexTemplateAction( - transportService, + mock(TransportService.class), null, - threadPool, + mock(ThreadPool.class), null, mock(ActionFilters.class), null @@ -704,9 +701,9 @@ public void testHandlerCorrectness() { ); var putComponentAction = new TransportPutComponentTemplateAction( - transportService, + mock(TransportService.class), null, - threadPool, + mock(ThreadPool.class), null, mock(ActionFilters.class), null, @@ -719,9 +716,9 @@ public void testHandlerCorrectness() { ); var delComponentAction = new TransportDeleteComponentTemplateAction( - transportService, + mock(TransportService.class), null, - threadPool, + mock(ThreadPool.class), null, mock(ActionFilters.class), null @@ -924,12 +921,10 @@ public void testTemplatesWithReservedPrefix() throws Exception { PutComposableIndexTemplateAction.Request pr = new PutComposableIndexTemplateAction.Request(conflictingTemplateName); - final ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putTemplateAction = new TransportPutComposableIndexTemplateAction( - transportService, + mock(TransportService.class), null, - threadPool, + mock(ThreadPool.class), null, mock(ActionFilters.class), null diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index 1b0c24664be31..ff96f4a00b883 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; @@ -31,7 +32,6 @@ import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -42,6 +42,7 @@ import java.util.function.Function; import static java.util.Collections.emptySet; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -106,6 +107,9 @@ private void indicesThatCannotBeCreatedTestCase( when(clusterService.localNode()).thenReturn(localNode); when(localNode.isIngestNode()).thenReturn(randomBoolean()); + final ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver( new ThreadContext(Settings.EMPTY), EmptySystemIndices.INSTANCE @@ -116,11 +120,9 @@ public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) } }; - final ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportBulkAction action = new TransportBulkAction( threadPool, - transportService, + mock(TransportService.class), clusterService, null, null, diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 85a74174b094f..fd84a1e5fe816 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -183,11 +183,6 @@ public void setupAction() { MockitoAnnotations.openMocks(this); // setup services that will be called by action transportService = mock(TransportService.class); - - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - clusterService = mock(ClusterService.class); localIngest = true; // setup nodes for local and remote diff --git a/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java b/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java index c2ff59f0ccbe0..93c4ea98adcd3 100644 --- a/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -48,8 +47,7 @@ public void testRejectImmutableConflictClusterStateUpdate() { ClusterService clusterService = mock(ClusterService.class); doReturn(clusterState).when(clusterService).state(); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - Action handler = new Action("internal:testAction", clusterService, transportService, mock(ActionFilters.class)); + Action handler = new Action("internal:testAction", clusterService, mock(TransportService.class), mock(ActionFilters.class)); // nothing should happen here, since the request doesn't touch any of the immutable state keys var future = new PlainActionFuture(); @@ -63,7 +61,7 @@ public void testRejectImmutableConflictClusterStateUpdate() { FakeReservedStateAwareAction action = new FakeReservedStateAwareAction( "internal:testClusterSettings", clusterService, - transportService, + mock(TransportService.class), mock(ActionFilters.class), null ); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java index d14429647c7d3..e45fc51cf6bdd 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.engine.DocIdSeqNoAndSource; @@ -32,6 +31,7 @@ import org.elasticsearch.index.shard.ReplicationGroup; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -67,7 +67,7 @@ public void setUp() throws Exception { transportService.acceptIncomingRequests(); transportService.registerRequestHandler( TransportUnpromotableShardRefreshAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, UnpromotableShardRefreshRequest::new, (request, channel, task) -> { unpromotableRefreshRequestReceived.set(true); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java index 7726feaa30868..8e95277bdc84f 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -424,7 +424,7 @@ protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentF ), TransportWriteActionTests.this.clusterService, null, - TransportWriteActionTests.threadPool, + null, null, new ActionFilters(new HashSet<>()), TestRequest::new, diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java index a18e7e8ce46f9..66503bf4cea79 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.NodeRoles; import org.elasticsearch.transport.TransportService; @@ -39,9 +38,8 @@ public void testDuplicateRecoveries() throws IOException { final ClusterService clusterService = mock(ClusterService.class); when(clusterService.getSettings()).thenReturn(NodeRoles.dataNode()); when(indicesService.clusterService()).thenReturn(clusterService); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService( - transportService, + mock(TransportService.class), indicesService, new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), mock(RecoveryPlannerService.class) diff --git a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java index 56e72c25802e3..4ffd1e3ed94eb 100644 --- a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java +++ b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java @@ -132,7 +132,7 @@ private void runTest( childTransportService.getTaskManager().setTaskCancellationService(new TaskCancellationService(childTransportService)); childTransportService.registerRequestHandler( "internal:testAction[c]", - threadPool.executor(ThreadPool.Names.MANAGEMENT), // busy-wait for cancellation but not on a transport thread + ThreadPool.Names.MANAGEMENT, // busy-wait for cancellation but not on a transport thread (StreamInput in) -> new TransportRequest.Empty(in) { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 2d0956fae5499..5635e07000029 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -35,7 +35,6 @@ import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.SuppressForbidden; @@ -126,7 +125,7 @@ public static MockTransportService startTransport( try { newService.registerRequestHandler( SearchShardsAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, SearchShardsRequest::new, (request, channel, task) -> { if ("index_not_found".equals(request.preference())) { @@ -136,50 +135,41 @@ public static MockTransportService startTransport( } } ); - newService.registerRequestHandler( - SearchAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - SearchRequest::new, - (request, channel, task) -> { - if ("index_not_found".equals(request.preference())) { - channel.sendResponse(new IndexNotFoundException("index")); - return; - } - SearchHits searchHits; - if ("null_target".equals(request.preference())) { - searchHits = new SearchHits( - new SearchHit[] { new SearchHit(0) }, - new TotalHits(1, TotalHits.Relation.EQUAL_TO), - 1F - ); - } else { - searchHits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN); - } - InternalSearchResponse response = new InternalSearchResponse( - searchHits, - InternalAggregations.EMPTY, - null, - null, - false, - null, - 1 - ); - SearchResponse searchResponse = new SearchResponse( - response, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); - channel.sendResponse(searchResponse); + newService.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { + if ("index_not_found".equals(request.preference())) { + channel.sendResponse(new IndexNotFoundException("index")); + return; } - ); + SearchHits searchHits; + if ("null_target".equals(request.preference())) { + searchHits = new SearchHits(new SearchHit[] { new SearchHit(0) }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1F); + } else { + searchHits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN); + } + InternalSearchResponse response = new InternalSearchResponse( + searchHits, + InternalAggregations.EMPTY, + null, + null, + false, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + response, + null, + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + channel.sendResponse(searchResponse); + }); newService.registerRequestHandler( ClusterStateAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); @@ -193,7 +183,7 @@ public static MockTransportService startTransport( if (RemoteClusterPortSettings.REMOTE_CLUSTER_SERVER_ENABLED.get(s)) { newService.registerRequestHandler( RemoteClusterNodesAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(knownNodes)) ); diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java index 84e4249bb0ccb..577f5d2c7dc18 100644 --- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; @@ -121,7 +120,7 @@ public MockTransportService startTransport( try { newService.registerRequestHandler( ClusterStateAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); @@ -135,7 +134,7 @@ public MockTransportService startTransport( if (hasClusterCredentials) { newService.registerRequestHandler( RemoteClusterNodesAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(knownNodes)) ); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java index 32a5b5dec9597..704de4fb2276a 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.RefCounted; @@ -109,47 +108,32 @@ private MockTransportService buildService(VersionInformation version, TransportV } public void testSendMessage() throws InterruptedException { - serviceA.registerRequestHandler( - "internal:test", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - SimpleTestRequest::new, - (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_A"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - } - ); + serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_A"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + }); final boolean cancellable = randomBoolean(); TransportActionProxy.registerProxyAction(serviceA, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceA, nodeB); - serviceB.registerRequestHandler( - "internal:test", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - SimpleTestRequest::new, - (request, channel, task) -> { - assertThat(task instanceof CancellableTask, equalTo(cancellable)); - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_B"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - } - ); + serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { + assertThat(task instanceof CancellableTask, equalTo(cancellable)); + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_B"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + }); TransportActionProxy.registerProxyAction(serviceB, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceB, nodeC); - serviceC.registerRequestHandler( - "internal:test", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - SimpleTestRequest::new, - (request, channel, task) -> { - assertThat(task instanceof CancellableTask, equalTo(cancellable)); - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_C"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - } - ); + serviceC.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { + assertThat(task instanceof CancellableTask, equalTo(cancellable)); + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_C"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + }); TransportActionProxy.registerProxyAction(serviceC, "internal:test", cancellable, SimpleTestResponse::new); // Node A -> Node B -> Node C: different versions - serialize the response @@ -264,7 +248,7 @@ public void testSendLocalRequest() throws Exception { final boolean cancellable = randomBoolean(); serviceB.registerRequestHandler( "internal:test", - threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), + randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), SimpleTestRequest::new, (request, channel, task) -> { try { @@ -324,39 +308,24 @@ public void handleException(TransportException exp) { public void testException() throws InterruptedException { boolean cancellable = randomBoolean(); - serviceA.registerRequestHandler( - "internal:test", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - SimpleTestRequest::new, - (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - SimpleTestResponse response = new SimpleTestResponse("TS_A"); - channel.sendResponse(response); - } - ); + serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + SimpleTestResponse response = new SimpleTestResponse("TS_A"); + channel.sendResponse(response); + }); TransportActionProxy.registerProxyAction(serviceA, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceA, nodeB); - serviceB.registerRequestHandler( - "internal:test", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - SimpleTestRequest::new, - (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - SimpleTestResponse response = new SimpleTestResponse("TS_B"); - channel.sendResponse(response); - } - ); + serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + SimpleTestResponse response = new SimpleTestResponse("TS_B"); + channel.sendResponse(response); + }); TransportActionProxy.registerProxyAction(serviceB, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceB, nodeC); - serviceC.registerRequestHandler( - "internal:test", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - SimpleTestRequest::new, - (request, channel, task) -> { - throw new ElasticsearchException("greetings from TS_C"); - } - ); + serviceC.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { + throw new ElasticsearchException("greetings from TS_C"); + }); TransportActionProxy.registerProxyAction(serviceC, "internal:test", cancellable, SimpleTestResponse::new); CountDownLatch latch = new CountDownLatch(1); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java index 2eb77c706a3a2..b6d3ab5455fd6 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; @@ -69,7 +68,7 @@ protected void onSendRequest(long requestId, String action, TransportRequest req transportService.registerRequestHandler( testActionName, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, TransportRequest.Empty::new, (request, channel, task) -> channel.sendResponse(TransportResponse.Empty.INSTANCE) ); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java index c5aa918fdc0e2..6f422190e9870 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java @@ -155,7 +155,7 @@ public ExecutorService executor(String name) { for (final var executor : EXECUTOR_NAMES) { transportService.registerRequestHandler( ACTION_NAME_PREFIX + executor, - threadPool.executor(executor), + executor, TransportRequest.Empty::new, (request, channel, task) -> { if (randomBoolean()) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java b/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java deleted file mode 100644 index 4edca754df053..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.test; - -import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockingDetails; -import static org.mockito.Mockito.when; - -/** - * Utilities for setting up Mockito mocks. - */ -public class MockUtils { - - /** - * Sets up a mock TransportService that can answer calls to TransportService.getThreadPool().executor(String). - * - * @return A mocked TransportService instance - */ - public static TransportService setupTransportServiceWithThreadpoolExecutor() { - TransportService transportService = mock(TransportService.class); - ThreadPool threadPool = mock(ThreadPool.class); - - setMockReturns(transportService, threadPool); - - return transportService; - } - - /** - * Sets up a mock TransportService that can answer calls to TransportService.getThreadPool().executor(String), using the given - * threadPool in TransportService. - * - * @param threadPool A mock ThreadPool - * @return A mocked TransportService instance - */ - public static TransportService setupTransportServiceWithThreadpoolExecutor(ThreadPool threadPool) { - assert mockingDetails(threadPool).isMock(); - TransportService transportService = mock(TransportService.class); - - setMockReturns(transportService, threadPool); - - return transportService; - } - - /** - * Sets up the given mock TransportService so that it can answer calls to TransportService.getThreadPool().executor(String). - * - * @param transportService A mock TransportService to be set up. - */ - public static void setupTransportServiceWithThreadpoolExecutor(TransportService transportService) { - assert mockingDetails(transportService).isMock(); - ThreadPool threadPool = mock(ThreadPool.class); - - setMockReturns(transportService, threadPool); - } - - private static void setMockReturns(TransportService transportService, ThreadPool threadPool) { - when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index 2a80b4e2e7e45..4a683c3f7dd57 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -313,7 +313,7 @@ public void assertNoPendingHandshakes(Transport transport) { public void testHelloWorld() { serviceA.registerRequestHandler( "internal:sayHello", - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -397,7 +397,7 @@ public void testThreadContext() throws ExecutionException, InterruptedException serviceA.registerRequestHandler( "internal:ping_pong", - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, StringMessageRequest::new, (request, channel, task) -> { assertEquals("ping_user", threadPool.getThreadContext().getHeader("test.ping.user")); @@ -460,7 +460,7 @@ public void testLocalNodeConnection() throws InterruptedException { final AtomicReference exception = new AtomicReference<>(); serviceA.registerRequestHandler( "internal:localNode", - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, StringMessageRequest::new, (request, channel, task) -> { try { @@ -519,8 +519,8 @@ public void testMessageListeners() throws Exception { } }; final String ACTION = "internal:action"; - serviceA.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, requestHandler); - serviceB.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, requestHandler); + serviceA.registerRequestHandler(ACTION, ThreadPool.Names.GENERIC, TransportRequest.Empty::new, requestHandler); + serviceB.registerRequestHandler(ACTION, ThreadPool.Names.GENERIC, TransportRequest.Empty::new, requestHandler); class CountingListener implements TransportMessageListener { AtomicInteger requestsReceived = new AtomicInteger(); @@ -639,7 +639,7 @@ public void testVoidMessageCompressed() throws Exception { try (MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY)) { serviceA.registerRequestHandler( "internal:sayHello", - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, TransportRequest.Empty::new, (request, channel, task) -> { try { @@ -695,7 +695,7 @@ public void testHelloWorldCompressed() throws Exception { try (MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY)) { serviceA.registerRequestHandler( "internal:sayHello", - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -765,18 +765,8 @@ public void testIndexingDataCompression() throws Exception { fail(e.getMessage()); } }; - serviceA.registerRequestHandler( - "internal:sayHello", - threadPool.executor(ThreadPool.Names.GENERIC), - StringMessageRequest::new, - handler - ); - serviceC.registerRequestHandler( - "internal:sayHello", - threadPool.executor(ThreadPool.Names.GENERIC), - StringMessageRequest::new, - handler - ); + serviceA.registerRequestHandler("internal:sayHello", ThreadPool.Names.GENERIC, StringMessageRequest::new, handler); + serviceC.registerRequestHandler("internal:sayHello", ThreadPool.Names.GENERIC, StringMessageRequest::new, handler); Settings settingsWithCompress = Settings.builder() .put(TransportSettings.TRANSPORT_COMPRESS.getKey(), Compression.Enabled.INDEXING_DATA) @@ -840,7 +830,7 @@ public void handleException(TransportException exp) { public void testErrorMessage() throws InterruptedException { serviceA.registerRequestHandler( "internal:sayHelloException", - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -903,7 +893,7 @@ public void testConcurrentSendRespondAndDisconnect() throws BrokenBarrierExcepti Set responseErrors = ConcurrentCollections.newConcurrentSet(); serviceA.registerRequestHandler( "internal:test", - threadPool.executor(randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC), + randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC, TestRequest::new, (request, channel, task) -> { try { @@ -922,7 +912,7 @@ public void testConcurrentSendRespondAndDisconnect() throws BrokenBarrierExcepti logger.trace("caught exception while responding from node B", e); } }; - serviceB.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, TestRequest::new, ignoringRequestHandler); + serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, TestRequest::new, ignoringRequestHandler); int halfSenders = scaledRandomIntBetween(3, 10); final CyclicBarrier go = new CyclicBarrier(halfSenders * 2 + 1); @@ -1013,12 +1003,7 @@ public void onAfter() { // simulate restart of nodeB serviceB.close(); MockTransportService newService = buildService("TS_B_" + i, version1, transportVersion1, Settings.EMPTY); - newService.registerRequestHandler( - "internal:test", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TestRequest::new, - ignoringRequestHandler - ); + newService.registerRequestHandler("internal:test", ThreadPool.Names.SAME, TestRequest::new, ignoringRequestHandler); serviceB = newService; nodeB = newService.getLocalDiscoNode(); connectToNode(serviceB, nodeA); @@ -1042,7 +1027,7 @@ public void testNotifyOnShutdown() throws Exception { try { serviceA.registerRequestHandler( "internal:foobar", - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, StringMessageRequest::new, (request, channel, task) -> { try { @@ -1075,7 +1060,7 @@ public void testNotifyOnShutdown() throws Exception { public void testTimeoutSendExceptionWithNeverSendingBackResponse() throws Exception { serviceA.registerRequestHandler( "internal:sayHelloTimeoutNoResponse", - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, StringMessageRequest::new, (request, channel, task) -> assertThat("moshe", equalTo(request.message)) ); // don't send back a response @@ -1120,7 +1105,7 @@ public void testTimeoutSendExceptionWithDelayedResponse() throws Exception { Semaphore inFlight = new Semaphore(Integer.MAX_VALUE); serviceA.registerRequestHandler( "internal:sayHelloTimeoutDelayedResponse", - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, StringMessageRequest::new, (request, channel, task) -> { String message = request.message; @@ -1265,22 +1250,12 @@ public void handleResponse(StringMessageResponse response) {} public void handleException(TransportException exp) {} }; - serviceA.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); - serviceA.registerRequestHandler("internal:testNotSeen", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); - serviceA.registerRequestHandler( - "internal:testError", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - StringMessageRequest::new, - handlerWithError - ); - serviceB.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); - serviceB.registerRequestHandler("internal:testNotSeen", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); - serviceB.registerRequestHandler( - "internal:testError", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - StringMessageRequest::new, - handlerWithError - ); + serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, StringMessageRequest::new, handler); + serviceA.registerRequestHandler("internal:testNotSeen", ThreadPool.Names.SAME, StringMessageRequest::new, handler); + serviceA.registerRequestHandler("internal:testError", ThreadPool.Names.SAME, StringMessageRequest::new, handlerWithError); + serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, StringMessageRequest::new, handler); + serviceB.registerRequestHandler("internal:testNotSeen", ThreadPool.Names.SAME, StringMessageRequest::new, handler); + serviceB.registerRequestHandler("internal:testError", ThreadPool.Names.SAME, StringMessageRequest::new, handlerWithError); String includeSettings; String excludeSettings; @@ -1595,18 +1570,13 @@ public void writeTo(StreamOutput out) throws IOException { } public void testVersionFrom0to1() throws Exception { - serviceB.registerRequestHandler( - "internal:version", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - Version1Request::new, - (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - assertThat(request.value2, equalTo(0)); // not set, coming from service A - Version1Response response = new Version1Response(1, 2); - channel.sendResponse(response); - assertEquals(transportVersion0, channel.getVersion()); - } - ); + serviceB.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version1Request::new, (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + assertThat(request.value2, equalTo(0)); // not set, coming from service A + Version1Response response = new Version1Response(1, 2); + channel.sendResponse(response); + assertEquals(transportVersion0, channel.getVersion()); + }); Version0Request version0Request = new Version0Request(); version0Request.value1 = 1; @@ -1643,17 +1613,12 @@ public void handleException(TransportException exp) { } public void testVersionFrom1to0() throws Exception { - serviceA.registerRequestHandler( - "internal:version", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - Version0Request::new, - (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - Version0Response response = new Version0Response(1); - channel.sendResponse(response); - assertEquals(transportVersion0, channel.getVersion()); - } - ); + serviceA.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version0Request::new, (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + Version0Response response = new Version0Response(1); + channel.sendResponse(response); + assertEquals(transportVersion0, channel.getVersion()); + }); Version1Request version1Request = new Version1Request(); version1Request.value1 = 1; @@ -1693,20 +1658,15 @@ public void handleException(TransportException exp) { } public void testVersionFrom1to1() throws Exception { - serviceB.registerRequestHandler( - "internal:version", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - Version1Request::new, - (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - assertThat(request.value2, equalTo(2)); - Version1Response response = new Version1Response(1, 2); - channel.sendResponse(response); - // channel versions don't make sense on DirectResponseChannel - assertThat(channel, instanceOf(TaskTransportChannel.class)); - assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); - } - ); + serviceB.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version1Request::new, (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + assertThat(request.value2, equalTo(2)); + Version1Response response = new Version1Response(1, 2); + channel.sendResponse(response); + // channel versions don't make sense on DirectResponseChannel + assertThat(channel, instanceOf(TaskTransportChannel.class)); + assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); + }); Version1Request version1Request = new Version1Request(); version1Request.value1 = 1; @@ -1746,19 +1706,14 @@ public void handleException(TransportException exp) { } public void testVersionFrom0to0() throws Exception { - serviceA.registerRequestHandler( - "internal:version", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - Version0Request::new, - (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - Version0Response response = new Version0Response(1); - channel.sendResponse(response); - // channel versions don't make sense on DirectResponseChannel - assertThat(channel, instanceOf(TaskTransportChannel.class)); - assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); - } - ); + serviceA.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version0Request::new, (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + Version0Response response = new Version0Response(1); + channel.sendResponse(response); + // channel versions don't make sense on DirectResponseChannel + assertThat(channel, instanceOf(TaskTransportChannel.class)); + assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); + }); Version0Request version0Request = new Version0Request(); version0Request.value1 = 1; @@ -1797,7 +1752,7 @@ public void handleException(TransportException exp) { public void testMockFailToSendNoConnectRule() throws Exception { serviceA.registerRequestHandler( "internal:sayHello", - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -1853,7 +1808,7 @@ public void handleException(TransportException exp) { public void testMockUnresponsiveRule() throws InterruptedException { serviceA.registerRequestHandler( "internal:sayHello", - threadPool.executor(ThreadPool.Names.GENERIC), + ThreadPool.Names.GENERIC, StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -1905,16 +1860,11 @@ public void testHostOnMessages() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(2); final AtomicReference addressA = new AtomicReference<>(); final AtomicReference addressB = new AtomicReference<>(); - serviceB.registerRequestHandler( - "internal:action1", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TestRequest::new, - (request, channel, task) -> { - addressA.set(request.remoteAddress()); - channel.sendResponse(new TestResponse((String) null)); - latch.countDown(); - } - ); + serviceB.registerRequestHandler("internal:action1", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { + addressA.set(request.remoteAddress()); + channel.sendResponse(new TestResponse((String) null)); + latch.countDown(); + }); serviceA.sendRequest(nodeB, "internal:action1", new TestRequest(), new TransportResponseHandler() { @Override public TestResponse read(StreamInput in) throws IOException { @@ -1952,15 +1902,10 @@ public void handleException(TransportException exp) { public void testRejectEarlyIncomingRequests() throws Exception { try (TransportService service = buildService("TS_TEST", version0, transportVersion0, null, Settings.EMPTY, false, false)) { AtomicBoolean requestProcessed = new AtomicBoolean(false); - service.registerRequestHandler( - "internal:action", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TestRequest::new, - (request, channel, task) -> { - requestProcessed.set(true); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - ); + service.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { + requestProcessed.set(true); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + }); DiscoveryNode node = service.getLocalNode(); serviceA.close(); @@ -2200,19 +2145,19 @@ public Executor executor(ThreadPool threadPool) { } serviceB.registerRequestHandler( "internal:action1", - threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), + randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), TestRequest::new, new TestRequestHandler(serviceB) ); serviceC.registerRequestHandler( "internal:action1", - threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), + randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), TestRequest::new, new TestRequestHandler(serviceC) ); serviceA.registerRequestHandler( "internal:action1", - threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), + randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), TestRequest::new, new TestRequestHandler(serviceA) ); @@ -2286,7 +2231,7 @@ public Executor executor(ThreadPool threadPool) { public void testRegisterHandlerTwice() { serviceB.registerRequestHandler( "internal:action1", - threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), + randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2296,7 +2241,7 @@ public void testRegisterHandlerTwice() { IllegalArgumentException.class, () -> serviceB.registerRequestHandler( "internal:action1", - threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), + randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2306,7 +2251,7 @@ public void testRegisterHandlerTwice() { serviceA.registerRequestHandler( "internal:action1", - threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), + randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2504,21 +2449,16 @@ public void run() { public void testResponseHeadersArePreserved() throws InterruptedException { List executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet()); CollectionUtil.timSort(executors); // makes sure it's reproducible - serviceA.registerRequestHandler( - "internal:action", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TestRequest::new, - (request, channel, task) -> { + serviceA.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - threadPool.getThreadContext().putTransient("boom", new Object()); - threadPool.getThreadContext().addResponseHeader("foo.bar", "baz"); - if ("fail".equals(request.info)) { - throw new RuntimeException("boom"); - } else { - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } + threadPool.getThreadContext().putTransient("boom", new Object()); + threadPool.getThreadContext().addResponseHeader("foo.bar", "baz"); + if ("fail".equals(request.info)) { + throw new RuntimeException("boom"); + } else { + channel.sendResponse(TransportResponse.Empty.INSTANCE); } - ); + }); CountDownLatch latch = new CountDownLatch(2); @@ -2572,14 +2512,9 @@ public void testHandlerIsInvokedOnConnectionClose() throws IOException, Interrup List executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet()); CollectionUtil.timSort(executors); // makes sure it's reproducible TransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); - serviceC.registerRequestHandler( - "internal:action", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TestRequest::new, - (request, channel, task) -> { - // do nothing - } - ); + serviceC.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { + // do nothing + }); CountDownLatch latch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler() { @Override @@ -2641,31 +2576,26 @@ public void testConcurrentDisconnectOnNonPublishedConnection() throws IOExceptio MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); CountDownLatch receivedLatch = new CountDownLatch(1); CountDownLatch sendResponseLatch = new CountDownLatch(1); - serviceC.registerRequestHandler( - "internal:action", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TestRequest::new, - (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); - } + serviceC.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); } + } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - }); - } - ); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + }); + }); CountDownLatch responseLatch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @Override @@ -2713,31 +2643,26 @@ public void testTransportStats() throws Exception { MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); CountDownLatch receivedLatch = new CountDownLatch(1); CountDownLatch sendResponseLatch = new CountDownLatch(1); - serviceB.registerRequestHandler( - "internal:action", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TestRequest::new, - (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); - } + serviceB.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); } + } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - }); - } - ); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + }); + }); CountDownLatch responseLatch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @Override @@ -2830,31 +2755,26 @@ public void testTransportStatsWithException() throws Exception { CountDownLatch sendResponseLatch = new CountDownLatch(1); Exception ex = new RuntimeException("boom"); ex.setStackTrace(new StackTraceElement[0]); - serviceB.registerRequestHandler( - "internal:action", - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TestRequest::new, - (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); - } + serviceB.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); } + } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - onFailure(ex); - } - }); - } - ); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + onFailure(ex); + } + }); + }); CountDownLatch responseLatch = new CountDownLatch(1); AtomicReference receivedException = new AtomicReference<>(null); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @@ -3200,41 +3120,31 @@ public void testFailToSendIllegalStateException() throws InterruptedException { public void testChannelToString() { final String ACTION = "internal:action"; - serviceA.registerRequestHandler( - ACTION, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TransportRequest.Empty::new, - (request, channel, task) -> { - assertThat( - channel.toString(), - allOf( - containsString("DirectResponseChannel"), - containsString('{' + ACTION + '}'), - containsString("TaskTransportChannel{task=" + task.getId() + '}') - ) - ); - assertThat(new ChannelActionListener<>(channel).toString(), containsString(channel.toString())); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - ); - serviceB.registerRequestHandler( - ACTION, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TransportRequest.Empty::new, - (request, channel, task) -> { - assertThat( - channel.toString(), - allOf( - containsString("TcpTransportChannel"), - containsString('{' + ACTION + '}'), - containsString("TaskTransportChannel{task=" + task.getId() + '}'), - containsString("localAddress="), - containsString(serviceB.getLocalNode().getAddress().toString()) - ) - ); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - ); + serviceA.registerRequestHandler(ACTION, ThreadPool.Names.SAME, TransportRequest.Empty::new, (request, channel, task) -> { + assertThat( + channel.toString(), + allOf( + containsString("DirectResponseChannel"), + containsString('{' + ACTION + '}'), + containsString("TaskTransportChannel{task=" + task.getId() + '}') + ) + ); + assertThat(new ChannelActionListener<>(channel).toString(), containsString(channel.toString())); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + }); + serviceB.registerRequestHandler(ACTION, ThreadPool.Names.SAME, TransportRequest.Empty::new, (request, channel, task) -> { + assertThat( + channel.toString(), + allOf( + containsString("TcpTransportChannel"), + containsString('{' + ACTION + '}'), + containsString("TaskTransportChannel{task=" + task.getId() + '}'), + containsString("localAddress="), + containsString(serviceB.getLocalNode().getAddress().toString()) + ) + ); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + }); PlainActionFuture.get( f -> submitRequest( @@ -3323,7 +3233,7 @@ public void writeTo(StreamOutput out) throws IOException { final var requestSize = between(0, ByteSizeUnit.MB.toIntBytes(1)); final var responseSize = between(0, ByteSizeUnit.MB.toIntBytes(1)); - serviceB.registerRequestHandler(ACTION, EsExecutors.DIRECT_EXECUTOR_SERVICE, Request::new, (request, channel, task) -> { + serviceB.registerRequestHandler(ACTION, ThreadPool.Names.SAME, Request::new, (request, channel, task) -> { assertEquals(requestSize, request.refSize); channel.sendResponse(new Response(responseSize)); }); diff --git a/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java index 6d5542db5e529..2925254ea992b 100644 --- a/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java +++ b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java @@ -322,12 +322,7 @@ public void handleException(TransportException exp) { } private void registerRequestHandler(TransportService transportService, TransportRequestHandler handler) { - transportService.registerRequestHandler( - TEST_ACTION, - transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), - TestRequest::new, - handler - ); + transportService.registerRequestHandler(TEST_ACTION, ThreadPool.Names.GENERIC, TestRequest::new, handler); } private void send( diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java index eb7cc9c51c62e..ee7924428dd0b 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -60,15 +59,13 @@ public void init() { } public void testAvailable() throws Exception { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); - AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(transportService, mock(ActionFilters.class)); + AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); Client client = mockClient(); AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction( - transportService, + mock(TransportService.class), clusterService, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), null, client @@ -87,16 +84,14 @@ public void testAvailable() throws Exception { } public void testEnabled() throws Exception { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); - AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(transportService, mock(ActionFilters.class)); + AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); assertTrue(featureSet.enabled()); Client client = mockClient(); AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction( - transportService, + mock(TransportService.class), clusterService, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), null, client diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java index 6b0b53aedd12d..4f9d803c50c25 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; @@ -37,7 +36,6 @@ import static java.util.stream.Collectors.toList; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -46,10 +44,6 @@ public TransportAnalyticsStatsAction action(AnalyticsUsage usage) { TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - ClusterService clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); when(clusterService.localNode()).thenReturn(discoveryNode); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java index 91dc4f7d9d479..93ac8bdc85896 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; @@ -40,12 +39,10 @@ public class TransportDeleteAutoscalingPolicyActionTests extends AutoscalingTestCase { public void testWriteBlock() { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportDeleteAutoscalingPolicyAction action = new TransportDeleteAutoscalingPolicyAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); @@ -64,12 +61,10 @@ public void testWriteBlock() { } public void testNoWriteBlock() { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportDeleteAutoscalingPolicyAction action = new TransportDeleteAutoscalingPolicyAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java index 12d9fcee8bd10..17f97e2ddc050 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingLicenseChecker; @@ -38,12 +37,10 @@ public class TransportGetAutoscalingPolicyActionTests extends AutoscalingTestCase { public void testReadBlock() { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportGetAutoscalingPolicyAction action = new TransportGetAutoscalingPolicyAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), new AutoscalingLicenseChecker(() -> true) @@ -67,12 +64,10 @@ public void testReadBlock() { } public void testNoReadBlock() { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportGetAutoscalingPolicyAction action = new TransportGetAutoscalingPolicyAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), new AutoscalingLicenseChecker(() -> true) diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java index bca2fc4ab96af..a39b3113a96ca 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingLicenseChecker; @@ -42,12 +41,10 @@ public class TransportPutAutoscalingPolicyActionTests extends AutoscalingTestCas private static final PolicyValidator NO_VALIDATION = policy -> {}; public void testWriteBlock() { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportPutAutoscalingPolicyAction action = new TransportPutAutoscalingPolicyAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_VALIDATION, @@ -68,12 +65,10 @@ public void testWriteBlock() { } public void testNoWriteBlock() { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportPutAutoscalingPolicyAction action = new TransportPutAutoscalingPolicyAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_VALIDATION, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java index 1d81549f7adec..7a7d375781e16 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; @@ -49,9 +48,8 @@ public void init() { } public void testAvailable() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -66,9 +64,8 @@ public void testAvailable() { public void testEnabled() { Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState @@ -76,14 +73,13 @@ public void testEnabled() { assertThat(featureSet.enabled(), equalTo(false)); settings = Settings.builder().put("xpack.ccr.enabled", true); - featureSet = new CCRInfoTransportAction(transportService, mock(ActionFilters.class), settings.build(), licenseState); + featureSet = new CCRInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState); assertThat(featureSet.enabled(), equalTo(true)); } public void testName() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -141,12 +137,10 @@ public void testUsageStats() throws Exception { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build(); Mockito.when(clusterService.state()).thenReturn(clusterState); - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new CCRUsageTransportAction( - transportService, + mock(TransportService.class), null, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), null, Settings.EMPTY, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java index f2423ca4d1d16..95ab1c2323f7d 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; @@ -58,7 +57,6 @@ public void testActionNames() { final TransportService transportService = mock(TransportService.class); final CcrRestoreSourceService ccrRestoreSourceService = mock(CcrRestoreSourceService.class); - MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new ClearCcrRestoreSessionAction.TransportAction(actionFilters, transportService, ccrRestoreSourceService); assertThat(action.actionName, equalTo(ClearCcrRestoreSessionAction.NAME)); @@ -89,7 +87,6 @@ public void testRequestedShardIdMustBeConsistentWithSessionShardId() { } }).when(ccrRestoreSourceService).ensureSessionShardIdConsistency(anyString(), any()); - MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new ClearCcrRestoreSessionAction.TransportAction(actionFilters, transportService, ccrRestoreSourceService); final String sessionUUID = UUIDs.randomBase64UUID(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java index 307bb6e67b10f..7357873bba0f2 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; @@ -63,7 +62,6 @@ public void testActionNames() { final TransportService transportService = mock(TransportService.class); final CcrRestoreSourceService ccrRestoreSourceService = mock(CcrRestoreSourceService.class); - MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new GetCcrRestoreFileChunkAction.TransportAction( bigArrays, transportService, @@ -105,7 +103,6 @@ public void testRequestedShardIdMustBeConsistentWithSessionShardId() { } }).when(ccrRestoreSourceService).ensureSessionShardIdConsistency(anyString(), any()); - MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new GetCcrRestoreFileChunkAction.TransportAction( bigArrays, transportService, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java index 1c09fcb746f49..7eca64b82cbb4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java @@ -135,7 +135,7 @@ public TransportTermsEnumAction( transportService.registerRequestHandler( transportShardAction, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, NodeTermsEnumRequest::new, new NodeTransportHandler() ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java index b457a77e38f01..0801ae1834954 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import java.util.ArrayList; @@ -60,9 +59,8 @@ public void testDoExecute() throws Exception { }); } - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); TransportXPackInfoAction action = new TransportXPackInfoAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), licenseService, client diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java index 190ac8d388bcf..e60f8c1ccae6e 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ESTestCase; @@ -34,7 +33,6 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; public class TransportNodeDeprecationCheckActionTests extends ESTestCase { @@ -64,11 +62,6 @@ public void testNodeOperation() { DiscoveryNode node = Mockito.mock(DiscoveryNode.class); when(node.getId()).thenReturn("mock-node"); TransportService transportService = Mockito.mock(TransportService.class); - - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - when(transportService.getLocalNode()).thenReturn(node); PluginsService pluginsService = Mockito.mock(PluginsService.class); ActionFilters actionFilters = Mockito.mock(ActionFilters.class); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java index 76577b12cd8d9..89ff330051418 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -29,8 +28,7 @@ public class InternalExecutePolicyActionTests extends ESTestCase { @Before public void instantiateTransportAction() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - transportAction = new InternalExecutePolicyAction.Transport(transportService, mock(ActionFilters.class), null, null); + transportAction = new InternalExecutePolicyAction.Transport(mock(TransportService.class), mock(ActionFilters.class), null, null); } public void testSelectNodeForPolicyExecution() { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java index c9d2f97eb5624..c350465f0f725 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -55,14 +54,12 @@ public void init() throws Exception { } public void testAvailable() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - EqlInfoTransportAction featureSet = new EqlInfoTransportAction(transportService, mock(ActionFilters.class)); + EqlInfoTransportAction featureSet = new EqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - EqlInfoTransportAction featureSet = new EqlInfoTransportAction(transportService, mock(ActionFilters.class)); + EqlInfoTransportAction featureSet = new EqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } @@ -96,12 +93,10 @@ public void testUsageStats() throws Exception { when(mockNode.getId()).thenReturn("mocknode"); when(clusterService.localNode()).thenReturn(mockNode); - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new EqlUsageTransportAction( - transportService, + mock(TransportService.class), clusterService, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), null, client diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java index 53d5a66de7b66..8a39bc5110c93 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java @@ -36,9 +36,9 @@ public class DriverTaskRunner { public static final String ACTION_NAME = "internal:data/read/esql/compute"; private final TransportService transportService; - public DriverTaskRunner(TransportService transportService, Executor executor) { + public DriverTaskRunner(TransportService transportService, String executorName) { this.transportService = transportService; - transportService.registerRequestHandler(ACTION_NAME, executor, DriverRequest::new, new DriverRequestHandler()); + transportService.registerRequestHandler(ACTION_NAME, executorName, DriverRequest::new, new DriverRequestHandler()); } public void executeDrivers(Task parentTask, List drivers, Executor executor, ActionListener listener) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java index 6db4a8c4fe37d..c3efd67579d66 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java @@ -79,13 +79,13 @@ public ExchangeService(Settings settings, ThreadPool threadPool, String executor public void registerTransportHandler(TransportService transportService) { transportService.registerRequestHandler( EXCHANGE_ACTION_NAME, - threadPool.executor(requestExecutorName), + requestExecutorName, ExchangeRequest::new, new ExchangeTransportAction() ); transportService.registerRequestHandler( OPEN_EXCHANGE_ACTION_NAME, - threadPool.executor(requestExecutorName), + requestExecutorName, OpenExchangeRequest::new, new OpenExchangeRequestHandler() ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 4d783b9a1012c..8aff85d088d6f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -102,7 +102,12 @@ public EnrichLookupService(ClusterService clusterService, SearchService searchSe this.searchService = searchService; this.transportService = transportService; this.executor = transportService.getThreadPool().executor(EsqlPlugin.ESQL_THREAD_POOL_NAME); - transportService.registerRequestHandler(LOOKUP_ACTION_NAME, this.executor, LookupRequest::new, new TransportHandler()); + transportService.registerRequestHandler( + LOOKUP_ACTION_NAME, + EsqlPlugin.ESQL_THREAD_POOL_NAME, + LookupRequest::new, + new TransportHandler() + ); } public void lookupAsync( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index 27634bf0d4eaa..f725e4fe60175 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -103,9 +103,14 @@ public ComputeService( this.searchService = searchService; this.transportService = transportService; this.bigArrays = bigArrays.withCircuitBreaking(); + transportService.registerRequestHandler( + DATA_ACTION_NAME, + ESQL_THREAD_POOL_NAME, + DataNodeRequest::new, + new DataNodeRequestHandler() + ); this.esqlExecutor = threadPool.executor(ESQL_THREAD_POOL_NAME); - transportService.registerRequestHandler(DATA_ACTION_NAME, this.esqlExecutor, DataNodeRequest::new, new DataNodeRequestHandler()); - this.driverRunner = new DriverTaskRunner(transportService, this.esqlExecutor); + this.driverRunner = new DriverTaskRunner(transportService, ESQL_THREAD_POOL_NAME); this.exchangeService = exchangeService; this.enrichLookupService = enrichLookupService; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java index f0b3a89a4444b..7f0bdbab6add3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java @@ -43,17 +43,11 @@ public class EsqlInfoTransportActionTests extends ESTestCase { private ThreadPool threadPool; - private TransportService transportService; private Client client; @Before public void init() { threadPool = new TestThreadPool(getTestName()); - - // TODO: temporary, remove in #97879 - transportService = mock(TransportService.class); - when(transportService.getThreadPool()).thenReturn(threadPool); - client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); } @@ -64,12 +58,12 @@ public void shutdown() { } public void testAvailable() { - EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(transportService, mock(ActionFilters.class)); + EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(transportService, mock(ActionFilters.class)); + EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } @@ -104,7 +98,7 @@ public void testUsageStats() throws Exception { when(clusterService.localNode()).thenReturn(mockNode); var usageAction = new EsqlUsageTransportAction( - transportService, + mock(TransportService.class), clusterService, threadPool, mock(ActionFilters.class), diff --git a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java b/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java index 90034c9620923..f1bad49abda68 100644 --- a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java +++ b/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -34,9 +33,8 @@ public void init() throws Exception { } public void testAvailable() throws Exception { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GraphInfoTransportAction featureSet = new GraphInfoTransportAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -46,7 +44,7 @@ public void testAvailable() throws Exception { assertThat(featureSet.available(), is(available)); var usageAction = new GraphUsageTransportAction( - transportService, + mock(TransportService.class), null, mock(ThreadPool.class), mock(ActionFilters.class), @@ -75,10 +73,8 @@ public void testEnabled() throws Exception { } else { settings.put("xpack.graph.enabled", enabled); } - - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GraphInfoTransportAction featureSet = new GraphInfoTransportAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState @@ -86,7 +82,7 @@ public void testEnabled() throws Exception { assertThat(featureSet.enabled(), is(enabled)); GraphUsageTransportAction usageAction = new GraphUsageTransportAction( - transportService, + mock(TransportService.class), null, mock(ThreadPool.class), mock(ActionFilters.class), diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java index 0ea85bab36a4e..ae031a050d3d5 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider; import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument; @@ -58,10 +57,10 @@ public void setupMocks() { index = mock(SamlServiceProviderIndex.class); idp = mock(SamlIdentityProvider.class); when(idp.getAllowedNameIdFormats()).thenReturn(Set.of(TRANSIENT)); + now = Instant.ofEpochMilli(System.currentTimeMillis() + randomLongBetween(-500_000, 500_000)); final Clock clock = Clock.fixed(now, randomZone()); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - action = new TransportPutSamlServiceProviderAction(transportService, mock(ActionFilters.class), index, idp, clock); + action = new TransportPutSamlServiceProviderAction(mock(TransportService.class), mock(ActionFilters.class), index, idp, clock); } public void testRegisterNewServiceProvider() throws Exception { diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java index d81faf6a398d7..107421ce12a31 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; @@ -53,14 +52,18 @@ public void init() throws Exception { } public void testAvailable() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction(transportService, mock(ActionFilters.class)); + IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction( + mock(TransportService.class), + mock(ActionFilters.class) + ); assertThat(featureSet.available(), equalTo(true)); } public void testName() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction(transportService, mock(ActionFilters.class)); + IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction( + mock(TransportService.class), + mock(ActionFilters.class) + ); assertThat(featureSet.name(), equalTo("ilm")); } @@ -91,9 +94,13 @@ public void testUsageStats() throws Exception { ClusterState clusterState = buildClusterState(policies, indexPolicies); Mockito.when(clusterService.state()).thenReturn(clusterState); - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); - var usageAction = new IndexLifecycleUsageTransportAction(transportService, null, threadPool, mock(ActionFilters.class), null); + var usageAction = new IndexLifecycleUsageTransportAction( + mock(TransportService.class), + null, + mock(ThreadPool.class), + mock(ActionFilters.class), + null + ); PlainActionFuture future = new PlainActionFuture<>(); usageAction.masterOperation(null, null, clusterState, future); IndexLifecycleFeatureSetUsage ilmUsage = (IndexLifecycleFeatureSetUsage) future.get().getUsage(); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java index 3e0f19246dfe3..feb66c12c2613 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction; @@ -21,12 +20,10 @@ public class TransportDeleteLifecycleActionTests extends ESTestCase { public void testReservedStateHandler() { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportDeleteLifecycleAction putAction = new TransportDeleteLifecycleAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java index 8467e46630876..ef1d65f2e4950 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -47,12 +46,10 @@ public void testIsNoop() { } public void testReservedStateHandler() throws Exception { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportPutLifecycleAction putAction = new TransportPutLifecycleAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(NamedXContentRegistry.class), diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java index 2c19a79ac5af0..cdc50f28fc306 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.StopILMRequest; @@ -35,12 +34,10 @@ public class TransportStopILMActionTests extends ESTestCase { public void testStopILMClusterStatePriorityIsImmediate() { ClusterService clusterService = mock(ClusterService.class); - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportStopILMAction transportStopILMAction = new TransportStopILMAction( - transportService, + mock(TransportService.class), clusterService, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java index 56b7965be3686..d72d079d4ae19 100644 --- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java +++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportService; @@ -28,9 +27,8 @@ public class TransportDeletePipelineActionTests extends ESTestCase { public void testDeletePipelineWithMissingIndex() throws Exception { try (Client client = getFailureClient(new IndexNotFoundException("missing .logstash"))) { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportDeletePipelineAction action = new TransportDeletePipelineAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), client ); diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java index d8a4d048f1fe4..d8b4b167bcc46 100644 --- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java +++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLogAppender; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportService; @@ -101,8 +100,11 @@ public void onFailure(Exception e) { try (Client client = getMockClient(multiGetResponse)) { Loggers.addAppender(logger, mockLogAppender); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - TransportGetPipelineAction action = new TransportGetPipelineAction(transportService, mock(ActionFilters.class), client); + TransportGetPipelineAction action = new TransportGetPipelineAction( + mock(TransportService.class), + mock(ActionFilters.class), + client + ); action.doExecute(null, request, testActionListener); } finally { Loggers.removeAppender(logger, mockLogAppender); @@ -150,9 +152,8 @@ public void onFailure(Exception e) { } }; - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); try (Client client = getMockClient(searchResponse)) { - new TransportGetPipelineAction(transportService, mock(ActionFilters.class), client).doExecute( + new TransportGetPipelineAction(mock(TransportService.class), mock(ActionFilters.class), client).doExecute( null, request, testActionListener @@ -164,9 +165,8 @@ public void onFailure(Exception e) { public void testMissingIndexHandling() throws Exception { try (Client failureClient = getFailureClient(new IndexNotFoundException("foo"))) { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportGetPipelineAction action = new TransportGetPipelineAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), failureClient ); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java index 2ff9bb2906a5c..418d277977c16 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.ingest.IngestStats; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -146,12 +145,10 @@ private MachineLearningUsageTransportAction newUsageAction( boolean isDataFrameAnalyticsEnabled, boolean isNlpEnabled ) { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); return new MachineLearningUsageTransportAction( - transportService, + mock(TransportService.class), clusterService, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), TestEnvironment.newEnvironment(settings), @@ -165,9 +162,8 @@ private MachineLearningUsageTransportAction newUsageAction( } public void testAvailable() throws Exception { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); MachineLearningInfoTransportAction featureSet = new MachineLearningInfoTransportAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), commonSettings, licenseState @@ -195,11 +191,9 @@ public void testEnabled() throws Exception { enabled = randomBoolean(); settings.put("xpack.ml.enabled", enabled); } - - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); boolean expected = enabled; MachineLearningInfoTransportAction featureSet = new MachineLearningInfoTransportAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java index 67da449af850f..c55b78c07939a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.MlTasks; @@ -338,12 +337,10 @@ public static void addTask( } private TransportCloseJobAction createAction() { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); return new TransportCloseJobAction( - transportService, + mock(TransportService.class), client, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), clusterService, mock(AnomalyDetectionAuditor.class), diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java index 26409b1ada849..f85a13fcb3935 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java @@ -38,7 +38,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; public class TransportDeleteExpiredDataActionTests extends ESTestCase { @@ -60,10 +59,6 @@ public void remove(float requestsPerSec, ActionListener listener, Boole public void setup() { threadPool = new TestThreadPool("TransportDeleteExpiredDataActionTests thread pool"); TransportService transportService = mock(TransportService.class); - - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - Client client = mock(Client.class); ClusterService clusterService = mock(ClusterService.class); auditor = mock(AnomalyDetectionAuditor.class); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java index c8096a7b1f384..1c0243fcab098 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java @@ -81,12 +81,8 @@ public void testOperation() { } private TransportFinalizeJobExecutionAction createAction(ClusterService clusterService) { - // TODO: temporary, remove in #97879 - TransportService transportService = mock(TransportService.class); - when(transportService.getThreadPool()).thenReturn(threadPool); - return new TransportFinalizeJobExecutionAction( - transportService, + mock(TransportService.class), clusterService, threadPool, mock(ActionFilters.class), diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java index 4d242db394d10..79a376b90040f 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; @@ -45,14 +44,18 @@ public class MonitoringInfoTransportActionTests extends ESTestCase { private final Exporters exporters = mock(Exporters.class); public void testAvailable() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction(transportService, mock(ActionFilters.class)); + MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction( + mock(TransportService.class), + mock(ActionFilters.class) + ); assertThat(featureSet.available(), is(true)); } public void testMonitoringEnabledByDefault() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction(transportService, mock(ActionFilters.class)); + MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction( + mock(TransportService.class), + mock(ActionFilters.class) + ); assertThat(featureSet.enabled(), is(true)); } @@ -91,12 +94,10 @@ public void testUsage() throws Exception { when(exporters.getEnabledExporters()).thenReturn(exporterList); when(monitoring.isMonitoringActive()).thenReturn(collectionEnabled); - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new MonitoringUsageTransportAction( - transportService, + mock(TransportService.class), null, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), null, new MonitoringUsageServices(monitoring, exporters) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java index d2a694c04ae34..3676fc5d2e7a0 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -99,10 +98,6 @@ public void setUpMocks() { transportService = mock(TransportService.class); filters = mock(ActionFilters.class); - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - when(transportService.getTaskManager()).thenReturn(taskManager); when(taskManager.register(anyString(), eq(MonitoringBulkAction.NAME), any(TaskAwareRequest.class))).thenReturn(mock(Task.class)); when(filters.filters()).thenReturn(new ActionFilter[0]); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java index 9fe634a178179..dd78eb95e7f3d 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -50,6 +49,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportService; @@ -105,7 +105,7 @@ public TransportRollupSearchAction( transportService.registerRequestHandler( actionName, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, false, true, SearchRequest::new, diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java index b0881eb350d5a..ab7420f0b50ee 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -26,21 +25,23 @@ public class RollupInfoTransportActionTests extends ESTestCase { public void testAvailable() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - RollupInfoTransportAction featureSet = new RollupInfoTransportAction(transportService, mock(ActionFilters.class)); + RollupInfoTransportAction featureSet = new RollupInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabledDefault() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - RollupInfoTransportAction featureSet = new RollupInfoTransportAction(transportService, mock(ActionFilters.class)); + RollupInfoTransportAction featureSet = new RollupInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } public void testUsage() throws ExecutionException, InterruptedException, IOException { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); - var usageAction = new RollupUsageTransportAction(transportService, null, threadPool, mock(ActionFilters.class), null); + var usageAction = new RollupUsageTransportAction( + mock(TransportService.class), + null, + mock(ThreadPool.class), + mock(ActionFilters.class), + null + ); PlainActionFuture future = new PlainActionFuture<>(); usageAction.masterOperation(null, null, null, future); XPackFeatureSet.Usage rollupUsage = future.get().getUsage(); diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java index 985bec3ba8c77..ddba0395382f3 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -1109,7 +1108,7 @@ private static MockTransportService startTransport( try { service.registerRequestHandler( ClusterStateAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, ClusterStateRequest::new, (request, channel, task) -> { capturedHeaders.add( @@ -1122,7 +1121,7 @@ private static MockTransportService startTransport( ); service.registerRequestHandler( RemoteClusterNodesAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, RemoteClusterNodesAction.Request::new, (request, channel, task) -> { capturedHeaders.add( @@ -1133,7 +1132,7 @@ private static MockTransportService startTransport( ); service.registerRequestHandler( SearchShardsAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, SearchShardsRequest::new, (request, channel, task) -> { capturedHeaders.add( @@ -1142,36 +1141,31 @@ private static MockTransportService startTransport( channel.sendResponse(new SearchShardsResponse(List.of(), List.of(), Collections.emptyMap())); } ); - service.registerRequestHandler( - SearchAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - SearchRequest::new, - (request, channel, task) -> { - capturedHeaders.add( - new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders())) - ); - channel.sendResponse( - new SearchResponse( - new InternalSearchResponse( - new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - null, - false, - null, - 1 - ), + service.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { + capturedHeaders.add( + new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders())) + ); + channel.sendResponse( + new SearchResponse( + new InternalSearchResponse( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), + InternalAggregations.EMPTY, null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) - ); - } - ); + null, + false, + null, + 1 + ), + null, + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ) + ); + }); service.start(); service.acceptIncomingRequests(); success = true; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java index 88f233087e1dd..4b750fb7d2ab6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ToXContent; @@ -79,18 +78,24 @@ public void init() throws Exception { } public void testAvailable() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), settings); + SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction( + mock(TransportService.class), + mock(ActionFilters.class), + settings + ); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), settings); + SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction( + mock(TransportService.class), + mock(ActionFilters.class), + settings + ); assertThat(featureSet.enabled(), is(true)); Settings disabled = Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), false).build(); - featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), disabled); + featureSet = new SecurityInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class), disabled); assertThat(featureSet.enabled(), is(false)); } @@ -366,12 +371,10 @@ private void configureRoleMappingStoreUsage(boolean roleMappingStoreEnabled) { } private SecurityUsageTransportAction newUsageAction(Settings settings) { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); return new SecurityUsageTransportAction( - transportService, + mock(TransportService.class), null, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), null, settings, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java index f94acab50b6b5..e2d1fc3a24568 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyResponse; @@ -43,8 +42,12 @@ public void setUp() throws Exception { super.setUp(); apiKeyService = mock(ApiKeyService.class); securityContext = mock(SecurityContext.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - action = new TransportCreateCrossClusterApiKeyAction(transportService, mock(ActionFilters.class), apiKeyService, securityContext); + action = new TransportCreateCrossClusterApiKeyAction( + mock(TransportService.class), + mock(ActionFilters.class), + apiKeyService, + securityContext + ); } public void testApiKeyWillBeCreatedWithEmptyUserRoleDescriptors() throws IOException { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java index f9e219f168718..5d68dea7a146e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java @@ -61,7 +61,6 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; public class TransportGrantApiKeyActionTests extends ESTestCase { @@ -70,7 +69,6 @@ public class TransportGrantApiKeyActionTests extends ESTestCase { private ApiKeyUserRoleDescriptorResolver resolver; private AuthenticationService authenticationService; private ThreadPool threadPool; - private TransportService transportService; private AuthorizationService authorizationService; @Before @@ -83,12 +81,8 @@ public void setupMocks() throws Exception { threadPool = new TestThreadPool("TP-" + getTestName()); final ThreadContext threadContext = threadPool.getThreadContext(); - // TODO: temporary, remove in #97879 - TransportService transportService = mock(TransportService.class); - when(transportService.getThreadPool()).thenReturn(threadPool); - action = new TransportGrantApiKeyAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), threadContext, authenticationService, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java index 7ce920506d7d1..711c604206823 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.apikey.ApiKey; @@ -56,9 +55,8 @@ public void testExecute() throws IOException { () -> AuthenticationTestHelper.builder().build() ); when(securityContext.getAuthentication()).thenReturn(authentication); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportUpdateCrossClusterApiKeyAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), apiKeyService, securityContext @@ -122,9 +120,8 @@ public void testExecute() throws IOException { public void testAuthenticationCheck() { final SecurityContext securityContext = mock(SecurityContext.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportUpdateCrossClusterApiKeyAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), mock(ApiKeyService.class), securityContext diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java index dca8633d7c1eb..ed01d0ca6fc40 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java @@ -64,7 +64,6 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class TransportProfileHasPrivilegesActionTests extends ESTestCase { @@ -81,10 +80,6 @@ public class TransportProfileHasPrivilegesActionTests extends ESTestCase { public void setup() { threadPool = new TestThreadPool(TransportProfileHasPrivilegesActionTests.class.getSimpleName()); transportService = mock(TransportService.class); - - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - actionFilters = mock(ActionFilters.class); authorizationService = mock(AuthorizationService.class); nativePrivilegeStore = mock(NativePrivilegeStore.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java index 92bef206a9d6a..df414207da8ee 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest; @@ -51,13 +50,10 @@ public void setup() { fileRealm = mockRealm("file"); final Realms realms = mockRealms(List.of(nativeRealm, fileRealm)); - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); - action = new TransportClearRealmCacheAction( - threadPool, + mock(ThreadPool.class), mockClusterService(), - transportService, + mock(TransportService.class), mock(ActionFilters.class), realms, authenticationService diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java index 5194ff032c5ba..ac7680ea2b574 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.service.CreateServiceAccountTokenRequest; @@ -41,10 +40,8 @@ public class TransportCreateServiceAccountTokenActionTests extends ESTestCase { public void init() throws IOException { serviceAccountService = mock(ServiceAccountService.class); securityContext = mock(SecurityContext.class); - - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportCreateServiceAccountTokenAction = new TransportCreateServiceAccountTokenAction( - transportService, + mock(TransportService.class), new ActionFilters(Collections.emptySet()), serviceAccountService, securityContext diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java index 2050d31638fe9..8b0530e48c560 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.DeleteServiceAccountTokenRequest; import org.elasticsearch.xpack.core.security.action.service.DeleteServiceAccountTokenResponse; @@ -33,9 +32,8 @@ public class TransportDeleteServiceAccountTokenActionTests extends ESTestCase { @Before public void init() { serviceAccountService = mock(ServiceAccountService.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportDeleteServiceAccountTokenAction = new TransportDeleteServiceAccountTokenAction( - transportService, + mock(TransportService.class), new ActionFilters(Collections.emptySet()), serviceAccountService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java index e8d178efe6b4a..87ef1ea24559e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountRequest; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountResponse; @@ -32,9 +31,8 @@ public class TransportGetServiceAccountActionTests extends ESTestCase { @Before public void init() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportGetServiceAccountAction = new TransportGetServiceAccountAction( - transportService, + mock(TransportService.class), new ActionFilters(Collections.emptySet()) ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java index ec68ed7a2776f..da6508ac79ef5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountCredentialsRequest; @@ -59,10 +58,8 @@ public void init() throws UnknownHostException { when(transport.boundAddress()).thenReturn(new BoundTransportAddress(new TransportAddress[] { transportAddress }, transportAddress)); final Settings settings = builder.build(); serviceAccountService = mock(ServiceAccountService.class); - - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportGetServiceAccountCredentialsAction = new TransportGetServiceAccountCredentialsAction( - transportService, + mock(TransportService.class), new ActionFilters(Collections.emptySet()), serviceAccountService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java index dcdeace6b06b6..bd63dd65824bd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java @@ -88,7 +88,6 @@ public class TransportCreateTokenActionTests extends ESTestCase { .build(); private ThreadPool threadPool; - private TransportService transportService; private Client client; private SecurityIndexManager securityIndex; private ClusterService clusterService; @@ -100,11 +99,6 @@ public class TransportCreateTokenActionTests extends ESTestCase { @Before public void setupClient() { threadPool = new TestThreadPool(getTestName()); - - // TODO: temporary, remove in #97879 - transportService = mock(TransportService.class); - when(transportService.getThreadPool()).thenReturn(threadPool); - client = mock(Client.class); idxReqReference = new AtomicReference<>(); authenticationService = mock(AuthenticationService.class); @@ -227,7 +221,7 @@ public void testClientCredentialsCreatesWithoutRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - transportService, + mock(TransportService.class), new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -268,7 +262,7 @@ public void testPasswordGrantTypeCreatesWithRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - transportService, + mock(TransportService.class), new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -311,7 +305,7 @@ public void testKerberosGrantTypeCreatesWithRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - transportService, + mock(TransportService.class), new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -364,7 +358,7 @@ public void testKerberosGrantTypeWillFailOnBase64DecodeError() throws Exception final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - transportService, + mock(TransportService.class), new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -403,7 +397,7 @@ public void testServiceAccountCannotCreateOAuthToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - transportService, + mock(TransportService.class), new ActionFilters(Collections.emptySet()), tokenService, authenticationService, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java index f07bb7dc365ae..a24361078ff26 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java @@ -56,7 +56,6 @@ public class TransportInvalidateTokenActionTests extends ESTestCase { .build(); private ThreadPool threadPool; - private TransportService transportService; private Client client; private SecurityIndexManager securityIndex; private ClusterService clusterService; @@ -66,11 +65,6 @@ public class TransportInvalidateTokenActionTests extends ESTestCase { @Before public void setup() { threadPool = new TestThreadPool(getTestName()); - - // TODO: temporary, remove in #97879 - transportService = mock(TransportService.class); - when(transportService.getThreadPool()).thenReturn(threadPool); - securityContext = new SecurityContext(Settings.EMPTY, threadPool.getThreadContext()); client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); @@ -97,7 +91,7 @@ public void testInvalidateTokensWhenIndexUnavailable() throws Exception { clusterService ); final TransportInvalidateTokenAction action = new TransportInvalidateTokenAction( - transportService, + mock(TransportService.class), new ActionFilters(Collections.emptySet()), tokenService ); @@ -143,7 +137,7 @@ public void testInvalidateTokensWhenIndexClosed() throws Exception { clusterService ); final TransportInvalidateTokenAction action = new TransportInvalidateTokenAction( - transportService, + mock(TransportService.class), new ActionFilters(Collections.emptySet()), tokenService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java index 9f7815a1c9891..b4a79e070e927 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest; @@ -58,10 +57,8 @@ public void testHasPrivilegesRequestDoesNotAllowDLSRoleQueryBasedIndicesPrivileg .build(false); when(context.getAuthentication()).thenReturn(authentication); threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication); - - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportHasPrivilegesAction transportHasPrivilegesAction = new TransportHasPrivilegesAction( - transportService, + mock(TransportService.class), new ActionFilters(Set.of()), mock(AuthorizationService.class), mock(NativePrivilegeStore.class), @@ -113,10 +110,8 @@ public void testRequiresSameUser() { ); return null; }).when(authorizationService).checkPrivileges(any(), any(), anyCollection(), anyActionListener()); - - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportHasPrivilegesAction( - transportService, + mock(TransportService.class), new ActionFilters(Set.of()), authorizationService, privilegeStore, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java index 2a15aa09ddccd..600d4d1ba5fa4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.PageCacheRecycler; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.IOUtils; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.mocksocket.MockSocket; @@ -160,7 +159,7 @@ public TransportRequestHandler interceptHandler( DiscoveryNode remoteNode = remoteTransportService.getLocalDiscoNode(); remoteTransportService.registerRequestHandler( RemoteClusterNodesAction.NAME, - EsExecutors.DIRECT_EXECUTOR_SERVICE, + ThreadPool.Names.SAME, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(List.of(remoteNode))) ); diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java index cf28bf9922b24..67ce8c255fd94 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java @@ -19,8 +19,8 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.shutdown.TransportDeleteShutdownNodeAction.DeleteShutdownNodeExecutor; import org.elasticsearch.xpack.shutdown.TransportDeleteShutdownNodeAction.DeleteShutdownNodeTask; import org.junit.Before; @@ -55,7 +55,7 @@ public void init() { MockitoAnnotations.openMocks(this); // TODO: it takes almost 2 seconds to create these mocks....WHY?!? var threadPool = mock(ThreadPool.class); - var transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + var transportService = mock(TransportService.class); clusterService = mock(ClusterService.class); var actionFilters = mock(ActionFilters.class); var indexNameExpressionResolver = mock(IndexNameExpressionResolver.class); diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java index cbd51ceebc729..c264ad6ad1d03 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java @@ -19,8 +19,8 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.shutdown.TransportPutShutdownNodeAction.PutShutdownNodeExecutor; import org.elasticsearch.xpack.shutdown.TransportPutShutdownNodeAction.PutShutdownNodeTask; import org.junit.Before; @@ -61,7 +61,7 @@ public void init() { MockitoAnnotations.openMocks(this); // TODO: it takes almost 2 seconds to create these mocks....WHY?!? var threadPool = mock(ThreadPool.class); - var transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + var transportService = mock(TransportService.class); clusterService = mock(ClusterService.class); var actionFilters = mock(ActionFilters.class); var indexNameExpressionResolver = mock(IndexNameExpressionResolver.class); diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java index 954b5ba024418..4c71683a972a1 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.reservedstate.service.ReservedStateUpdateTask; import org.elasticsearch.reservedstate.service.ReservedStateUpdateTaskExecutor; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentParser; @@ -364,12 +363,10 @@ public void testOperatorControllerFromJSONContent() throws IOException { } public void testDeleteSLMReservedStateHandler() { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var deleteAction = new TransportDeleteSnapshotLifecycleAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); @@ -380,12 +377,10 @@ public void testDeleteSLMReservedStateHandler() { } public void testPutSLMReservedStateHandler() throws Exception { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putAction = new TransportPutSnapshotLifecycleAction( - transportService, + mock(TransportService.class), mock(ClusterService.class), - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java index 5d0e002c49036..0294c14f57494 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.action.StopILMAction; @@ -34,12 +33,11 @@ public class TransportStopSLMActionTests extends ESTestCase { public void testStopILMClusterStatePriorityIsImmediate() { ClusterService clusterService = mock(ClusterService.class); - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + TransportStopSLMAction transportStopSLMAction = new TransportStopSLMAction( - transportService, + mock(TransportService.class), clusterService, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java index ede64decbca33..83eb9d49a4e21 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -57,15 +56,13 @@ public void init() { } public void testAvailable() throws Exception { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); - SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(transportService, mock(ActionFilters.class)); + SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); var usageAction = new SpatialUsageTransportAction( - transportService, + mock(TransportService.class), clusterService, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), null, mockClient() @@ -83,16 +80,14 @@ public void testAvailable() throws Exception { } public void testEnabled() throws Exception { - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); - SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(transportService, mock(ActionFilters.class)); + SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); assertTrue(featureSet.enabled()); SpatialUsageTransportAction usageAction = new SpatialUsageTransportAction( - transportService, + mock(TransportService.class), clusterService, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), null, mockClient() diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java index 1b91f686a7d8d..4f42305984b57 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -48,8 +47,8 @@ public class SpatialStatsTransportActionTests extends ESTestCase { @Before public void mockServices() { + transportService = mock(TransportService.class); threadPool = mock(ThreadPool.class); - transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java index 01bad2c3dee60..f95fd337fe5b7 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -58,8 +57,11 @@ public void init() throws Exception { } public void testAvailable() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - SqlInfoTransportAction featureSet = new SqlInfoTransportAction(transportService, mock(ActionFilters.class), licenseState); + SqlInfoTransportAction featureSet = new SqlInfoTransportAction( + mock(TransportService.class), + mock(ActionFilters.class), + licenseState + ); assertThat(featureSet.available(), is(true)); } @@ -93,12 +95,10 @@ public void testUsageStats() throws Exception { when(mockNode.getId()).thenReturn("mocknode"); when(clusterService.localNode()).thenReturn(mockNode); - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new SqlUsageTransportAction( - transportService, + mock(TransportService.class), clusterService, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), null, licenseState, diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java index d2c23f768cef6..5c6f875415384 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; @@ -30,14 +29,12 @@ public class TransformInfoTransportActionTests extends ESTestCase { public void testAvailable() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - TransformInfoTransportAction featureSet = new TransformInfoTransportAction(transportService, mock(ActionFilters.class)); + TransformInfoTransportAction featureSet = new TransformInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabledDefault() { - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); - TransformInfoTransportAction featureSet = new TransformInfoTransportAction(transportService, mock(ActionFilters.class)); + TransformInfoTransportAction featureSet = new TransformInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); assertTrue(featureSet.enabled()); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java index ac683dca5bf26..0b796ee3d3f7d 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -56,23 +55,20 @@ public class WatcherInfoTransportActionTests extends ESTestCase { private MockLicenseState licenseState; private Client client; - private ThreadPool threadPool; - private TransportService transportService; @Before public void init() throws Exception { licenseState = mock(MockLicenseState.class); client = mock(Client.class); - threadPool = mock(ThreadPool.class); + ThreadPool threadPool = mock(ThreadPool.class); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); when(threadPool.getThreadContext()).thenReturn(threadContext); when(client.threadPool()).thenReturn(threadPool); - transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); } public void testAvailable() { WatcherInfoTransportAction featureSet = new WatcherInfoTransportAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -93,7 +89,7 @@ public void testEnabled() { settings.put("xpack.watcher.enabled", enabled); } WatcherInfoTransportAction featureSet = new WatcherInfoTransportAction( - transportService, + mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState @@ -134,9 +130,9 @@ public void testUsageStats() throws Exception { when(clusterService.localNode()).thenReturn(mockNode); var usageAction = new WatcherUsageTransportAction( - transportService, + mock(TransportService.class), clusterService, - threadPool, + mock(ThreadPool.class), mock(ActionFilters.class), null, Settings.EMPTY, diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java index 338f4e56f5663..79c93cd5b913a 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.license.TestUtils; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.watcher.WatcherMetadata; @@ -57,8 +56,8 @@ public class TransportAckWatchActionTests extends ESTestCase { @Before public void setupAction() { + TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); when(threadPool.getThreadContext()).thenReturn(threadContext); WatchParser watchParser = mock(WatchParser.class); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java index 95272619fac5d..09cf9daaf3685 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.license.TestUtils; import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ClientHelper; @@ -64,7 +63,7 @@ public void setupAction() throws Exception { ThreadPool threadPool = mock(ThreadPool.class); when(threadPool.getThreadContext()).thenReturn(threadContext); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + TransportService transportService = mock(TransportService.class); WatchParser parser = mock(WatchParser.class); when(parser.parseWithSecrets(eq("_id"), eq(false), any(), any(), any(), anyBoolean(), anyLong(), anyLong())).thenReturn(watch); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java index 6dfdf47a7a735..8d07d1eaf6043 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -45,8 +44,8 @@ public class TransportWatcherStatsActionTests extends ESTestCase { @Before public void setupTransportAction() { + TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); ClusterService clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); From 1efd3432334518f7dcaf56b0e732a82ccb39d1d6 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Fri, 8 Sep 2023 11:37:18 -0700 Subject: [PATCH 50/69] ESQL: Combine Evals into one expression (#99332) Simplify queries with multiple eval declarations --- .../src/main/resources/eval.csv-spec | 58 +++++++++++++++++++ .../esql/optimizer/LogicalPlanOptimizer.java | 22 ++++++- .../esql/optimizer/PhysicalPlanOptimizer.java | 5 +- .../xpack/esql/plan/logical/Eval.java | 7 ++- .../optimizer/LogicalPlanOptimizerTests.java | 32 +++++++--- .../optimizer/PhysicalPlanOptimizerTests.java | 33 +++++------ 6 files changed, 127 insertions(+), 30 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec index ba89685716059..f9d7a846dd99a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec @@ -108,3 +108,61 @@ from employees | eval true_bool = null is null, false_bool = null is not null, n true_bool:boolean | negated_true:boolean | false_bool:boolean | negated_false:boolean | first_name:keyword | last_name:keyword true | false | false | true | Georgi | Facello ; + + +repetitiveEval +from employees | keep emp_no | eval sum = emp_no + 1 +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no +| limit 3 +; + +emp_no:i | sum:i +10001 | 3230324 +10002 | 3230647 +10003 | 3230970 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index 4f6f523c53bbb..2e14c33c701cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -98,6 +98,7 @@ protected static List> rules() { var operators = new Batch<>( "Operator Optimization", new CombineProjections(), + new CombineEvals(), new PruneEmptyPlans(), new PropagateEmptyRelation(), new ConvertStringToByteRef(), @@ -312,6 +313,26 @@ private static Expression trimAliases(Expression e) { } } + /** + * Combine multiple Evals into one in order to reduce the number of nodes in a plan. + * TODO: eliminate unnecessary fields inside the eval as well + */ + static class CombineEvals extends OptimizerRules.OptimizerRule { + + CombineEvals() { + super(TransformDirection.UP); + } + + @Override + protected LogicalPlan rule(Eval eval) { + LogicalPlan plan = eval; + if (eval.child() instanceof Eval subEval) { + plan = new Eval(eval.source(), subEval.child(), CollectionUtils.combine(subEval.fields(), eval.fields())); + } + return plan; + } + } + // // Replace any reference attribute with its source, if it does not affect the result. // This avoids ulterior look-ups between attributes and its source across nodes. @@ -584,7 +605,6 @@ protected static class PushDownEval extends OptimizerRules.OptimizerRule { protected LogicalPlan rule(Eval eval) { LogicalPlan child = eval.child(); - // TODO: combine with CombineEval from https://github.com/elastic/elasticsearch-internal/pull/511 when merged if (child instanceof OrderBy orderBy) { return orderBy.replaceChild(eval.replaceChild(orderBy.child())); } else if (child instanceof Project) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java index b057dd8023031..3f05fa90ac8ab 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java @@ -20,8 +20,8 @@ import org.elasticsearch.xpack.esql.planner.PhysicalVerifier; import org.elasticsearch.xpack.ql.common.Failure; import org.elasticsearch.xpack.ql.expression.Alias; -import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.AttributeMap; +import org.elasticsearch.xpack.ql.expression.AttributeSet; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Literal; @@ -36,7 +36,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.LinkedHashSet; import java.util.List; import static java.lang.Boolean.FALSE; @@ -91,7 +90,7 @@ static class ProjectAwayColumns extends Rule { public PhysicalPlan apply(PhysicalPlan plan) { var projectAll = new Holder<>(TRUE); var keepCollecting = new Holder<>(TRUE); - var attributes = new LinkedHashSet(); + var attributes = new AttributeSet(); var aliases = new AttributeMap(); return plan.transformDown(UnaryExec.class, p -> { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java index f5000b4394e21..cb20d3a549915 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java @@ -23,6 +23,7 @@ public class Eval extends UnaryPlan { private final List fields; + private List lazyOutput; public Eval(Source source, LogicalPlan child, List fields) { super(source, child); @@ -35,7 +36,11 @@ public List fields() { @Override public List output() { - return mergeOutputAttributes(fields, child().output()); + if (lazyOutput == null) { + lazyOutput = mergeOutputAttributes(fields, child().output()); + } + + return lazyOutput; } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index b6d818038a72c..bd1615ef9e5f2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -255,6 +255,28 @@ public void testCombineProjectionWithPruning() { var from = as(agg.child(), EsRelation.class); } + /** + * Expects + * EsqlProject[[x{r}#3, y{r}#6]] + * \_Eval[[emp_no{f}#9 + 2[INTEGER] AS x, salary{f}#14 + 3[INTEGER] AS y]] + * \_Limit[10000[INTEGER]] + * \_EsRelation[test][_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g..] + */ + public void testCombineEvals() { + var plan = plan(""" + from test + | eval x = emp_no + 2 + | eval y = salary + 3 + | keep x, y + """); + + var project = as(plan, Project.class); + var eval = as(project.child(), Eval.class); + assertThat(Expressions.names(eval.fields()), contains("x", "y")); + var limit = as(eval.child(), Limit.class); + var source = as(limit.child(), EsRelation.class); + } + public void testCombineLimits() { var limitValues = new int[] { randomIntBetween(10, 99), randomIntBetween(100, 1000) }; var firstLimit = randomBoolean() ? 0 : 1; @@ -777,9 +799,7 @@ public void testCombineOrderByThroughEvalWithTwoDefs() { var topN = as(plan, TopN.class); assertThat(orderNames(topN), contains("z", "emp_no")); var eval = as(topN.child(), Eval.class); - assertThat(Expressions.names(eval.fields()), contains("z")); - eval = as(eval.child(), Eval.class); - assertThat(Expressions.names(eval.fields()), contains("x", "y")); + assertThat(Expressions.names(eval.fields()), contains("x", "y", "z")); as(eval.child(), EsRelation.class); } @@ -1599,8 +1619,6 @@ public void testNoPruningWhenDealingJustWithEvals() { """); var eval = as(plan, Eval.class); - eval = as(eval.child(), Eval.class); - eval = as(eval.child(), Eval.class); var limit = as(eval.child(), Limit.class); var agg = as(limit.child(), Aggregate.class); } @@ -1630,9 +1648,7 @@ public void testPruningDuplicateEvals() { var project = as(plan, Project.class); var eval = as(project.child(), Eval.class); - assertThat(Expressions.names(eval.fields()), contains("y")); - eval = as(eval.child(), Eval.class); - assertThat(Expressions.names(eval.fields()), contains("x")); + assertThat(Expressions.names(eval.fields()), contains("x", "y")); var limit = as(eval.child(), Limit.class); var source = as(limit.child(), EsRelation.class); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 83d71d2cc3ed1..87693bffc4433 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -366,14 +366,13 @@ public void testExtractorForField() { /** * Expected - * - * EvalExec[[emp_no{f}#538 + 1[INTEGER] AS emp_no]] - * \_EvalExec[[emp_no{f}#538 + 1[INTEGER] AS e]] - * \_LimitExec[10000[INTEGER]] - * \_ExchangeExec[GATHER,SINGLE_DISTRIBUTION] - * \_ProjectExec[[_meta_field{f}#537, emp_no{f}#538, first_name{f}#539, languages{f}#540, last_name{f}#541, salary{f}#542]] - * \_FieldExtractExec[_meta_field{f}#537, emp_no{f}#538, first_name{f}#53..] - * \_EsQueryExec[test], query[][_doc{f}#543], limit[10000] + * EvalExec[[emp_no{f}#7 + 1[INTEGER] AS e, emp_no{f}#7 + 1[INTEGER] AS emp_no]] + * \_LimitExec[10000[INTEGER]] + * \_ExchangeExec[[],false] + * \_ProjectExec[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, job{f}#14, job.raw{f}#15, languages{f}#10, last + * _name{f}#11, salary{f}#12]] + * \_FieldExtractExec[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsQueryExec[test], query[][_doc{f}#16], limit[10000], sort[] estimatedRowSize[324] */ public void testExtractorMultiEvalWithDifferentNames() { var plan = physicalPlan(""" @@ -383,8 +382,8 @@ public void testExtractorMultiEvalWithDifferentNames() { """); var optimized = optimizedPlan(plan); + var eval = as(optimized, EvalExec.class); - eval = as(eval.child(), EvalExec.class); var topLimit = as(eval.child(), LimitExec.class); var exchange = asRemoteExchange(topLimit.child()); var project = as(exchange.child(), ProjectExec.class); @@ -397,13 +396,13 @@ public void testExtractorMultiEvalWithDifferentNames() { /** * Expected - * EvalExec[[emp_no{r}#120 + 1[INTEGER] AS emp_no]] - * \_EvalExec[[emp_no{f}#125 + 1[INTEGER] AS emp_no]] - * \_LimitExec[10000[INTEGER]] - * \_ExchangeExec[GATHER,SINGLE_DISTRIBUTION] - * \_ProjectExec[[_meta_field{f}#124, emp_no{f}#125, first_name{f}#126, languages{f}#127, last_name{f}#128, salary{f}#129]] - * \_FieldExtractExec[_meta_field{f}#124, emp_no{f}#125, first_name{f}#12..] - * \_EsQueryExec[test], query[][_doc{f}#130], limit[10000] + * EvalExec[[emp_no{f}#7 + 1[INTEGER] AS emp_no, emp_no{r}#3 + 1[INTEGER] AS emp_no]] + * \_LimitExec[10000[INTEGER]] + * \_ExchangeExec[[],false] + * \_ProjectExec[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, job{f}#14, job.raw{f}#15, languages{f}#10, last + * _name{f}#11, salary{f}#12]] + * \_FieldExtractExec[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsQueryExec[test], query[][_doc{f}#16], limit[10000], sort[] estimatedRowSize[324] */ public void testExtractorMultiEvalWithSameName() { var plan = physicalPlan(""" @@ -413,8 +412,8 @@ public void testExtractorMultiEvalWithSameName() { """); var optimized = optimizedPlan(plan); + var eval = as(optimized, EvalExec.class); - eval = as(eval.child(), EvalExec.class); var topLimit = as(eval.child(), LimitExec.class); var exchange = asRemoteExchange(topLimit.child()); var project = as(exchange.child(), ProjectExec.class); From b7b733c0d35ab4268df67b4fba71fb4951f32f11 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 8 Sep 2023 15:21:41 -0400 Subject: [PATCH 51/69] Refactor WriteableIngestDocument (#99324) --- .../ingest/SimulateDocumentBaseResult.java | 13 +++++++---- .../ingest/SimulateProcessorResult.java | 14 ++++++++++- .../ingest/WriteableIngestDocument.java | 23 +++++++++++++++---- .../elasticsearch/ingest/IngestDocument.java | 15 +++++++++++- .../ingest/TrackingResultProcessor.java | 6 ++--- .../ingest/WriteableIngestDocumentTests.java | 9 ++++++++ .../ingest/IngestDocumentTests.java | 11 +++++++++ 7 files changed, 77 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java index 67379821ba014..9e4984d7497a3 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java @@ -50,12 +50,17 @@ public final class SimulateDocumentBaseResult implements SimulateDocumentResult } public SimulateDocumentBaseResult(IngestDocument ingestDocument) { + Exception failure = null; + WriteableIngestDocument wid = null; if (ingestDocument != null) { - this.ingestDocument = new WriteableIngestDocument(ingestDocument); - } else { - this.ingestDocument = null; + try { + wid = new WriteableIngestDocument(ingestDocument); + } catch (Exception ex) { + failure = ex; + } } - this.failure = null; + this.ingestDocument = wid; + this.failure = failure; } public SimulateDocumentBaseResult(Exception failure) { diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java index d4ae5e33f3b10..1d5776896ea19 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java @@ -126,7 +126,19 @@ public SimulateProcessorResult( ) { this.processorTag = processorTag; this.description = description; - this.ingestDocument = (ingestDocument == null) ? null : new WriteableIngestDocument(ingestDocument); + WriteableIngestDocument wid = null; + if (ingestDocument != null) { + try { + wid = new WriteableIngestDocument(ingestDocument); + } catch (Exception ex) { + // if there was a failure already, then track it as a suppressed exception + if (failure != null) { + ex.addSuppressed(failure); + } + failure = ex; + } + } + this.ingestDocument = wid; this.failure = failure; this.conditionalWithResult = conditionalWithResult; this.type = type; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java index 3a7e3c11fa141..40f8da6f7efaa 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java @@ -57,7 +57,7 @@ final class WriteableIngestDocument implements Writeable, ToXContentFragment { sourceAndMetadata.put(Metadata.VERSION_TYPE.getFieldName(), a[4]); } Map ingestMetadata = (Map) a[6]; - return new WriteableIngestDocument(new IngestDocument(sourceAndMetadata, ingestMetadata)); + return new WriteableIngestDocument(sourceAndMetadata, ingestMetadata); } ); static { @@ -83,17 +83,30 @@ final class WriteableIngestDocument implements Writeable, ToXContentFragment { PARSER.declareObject(constructorArg(), INGEST_DOC_PARSER, new ParseField(DOC_FIELD)); } + /** + * Builds a writeable ingest document that wraps a copy of the passed-in, non-null ingest document. + * + * @throws IllegalArgumentException if the passed-in ingest document references itself + */ WriteableIngestDocument(IngestDocument ingestDocument) { assert ingestDocument != null; - this.ingestDocument = ingestDocument; + this.ingestDocument = new IngestDocument(ingestDocument); // internal defensive copy } - WriteableIngestDocument(StreamInput in) throws IOException { - Map sourceAndMetadata = in.readMap(); - Map ingestMetadata = in.readMap(); + /** + * Builds a writeable ingest document by constructing the wrapped ingest document from the passed-in maps. + *

+ * This is intended for cases like deserialization, where we know the passed-in maps aren't self-referencing, + * and where a defensive copy is unnecessary. + */ + private WriteableIngestDocument(Map sourceAndMetadata, Map ingestMetadata) { this.ingestDocument = new IngestDocument(sourceAndMetadata, ingestMetadata); } + WriteableIngestDocument(StreamInput in) throws IOException { + this(in.readMap(), in.readMap()); + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeGenericMap(ingestDocument.getSourceAndMetadata()); diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 25827018e44a7..66931c7791bc8 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.VersionType; @@ -94,14 +95,26 @@ public IngestDocument(String index, String id, long version, String routing, Ver /** * Copy constructor that creates a new {@link IngestDocument} which has exactly the same properties as the one provided. + * + * @throws IllegalArgumentException if the passed-in ingest document references itself */ public IngestDocument(IngestDocument other) { this( - new IngestCtxMap(deepCopyMap(other.ctxMap.getSource()), other.ctxMap.getMetadata().clone()), + new IngestCtxMap(deepCopyMap(ensureNoSelfReferences(other.ctxMap.getSource())), other.ctxMap.getMetadata().clone()), deepCopyMap(other.ingestMetadata) ); } + /** + * Internal helper utility method to get around the issue that a {@code this(...) } constructor call must be the first statement + * in a constructor. This is only for use in the {@link IngestDocument#IngestDocument(IngestDocument)} copy constructor, it's not a + * general purpose method. + */ + private static Map ensureNoSelfReferences(Map source) { + CollectionUtils.ensureNoSelfReferences(source, null); + return source; + } + /** * Constructor to create an IngestDocument from its constituent maps. The maps are shallow copied. */ diff --git a/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java b/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java index f09b2c12c99dd..5bd811962a4c4 100644 --- a/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java @@ -98,7 +98,7 @@ public void execute(IngestDocument ingestDocument, BiConsumer { @@ -183,6 +184,14 @@ public void testXContentHashSetSerialization() throws Exception { } } + public void testCopiesTheIngestDocument() { + IngestDocument document = createRandomIngestDoc(); + WriteableIngestDocument wid = new WriteableIngestDocument(document); + + assertThat(wid.getIngestDocument(), equalTo(document)); + assertThat(wid.getIngestDocument(), not(sameInstance(document))); + } + static IngestDocument createRandomIngestDoc() { XContentType xContentType = randomFrom(XContentType.values()); BytesReference sourceBytes = RandomObjects.randomSource(random(), xContentType); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java index 924ca1fc7a1e9..ae46668352048 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java @@ -1069,6 +1069,17 @@ public void testCopyConstructor() { assertThat(ingestDocument.getFieldValue("_id", String.class), equalTo("bar1")); assertThat(ingestDocument.getFieldValue("hello", String.class), equalTo("world1")); } + + { + // the copy constructor rejects self-references + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + List someList = new ArrayList<>(); + someList.add("some string"); + someList.add(someList); // the list contains itself + ingestDocument.setFieldValue("someList", someList); + Exception e = expectThrows(IllegalArgumentException.class, () -> new IngestDocument(ingestDocument)); + assertThat(e.getMessage(), equalTo("Iterable object is self-referencing itself")); + } } public void testCopyConstructorWithZonedDateTime() { From 574fb0594654d3e136ca0bdab2c029360c5b9902 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 8 Sep 2023 22:18:35 +0200 Subject: [PATCH 52/69] Deduplicate org.apache.lucene.document.FieldType instances across mappers (#99361) We mostly have a handful of `FieldType` values here across all mappers and none of them contain attributes. There's only so many combinations here, lets deduplicate these to save some heap and set up subsequent mapper heap savings. --- .../extras/MatchOnlyTextFieldMapper.java | 17 ++++++------ .../mapper/extras/RankFeatureFieldMapper.java | 11 ++++---- .../extras/SearchAsYouTypeFieldMapper.java | 6 ++--- .../icu/ICUCollationKeywordFieldMapper.java | 2 +- .../AnnotatedTextFieldMapper.java | 2 +- .../mapper/murmur3/Murmur3FieldMapper.java | 7 ++--- .../index/mapper/CompletionFieldMapper.java | 15 ++++++----- .../index/mapper/CustomDocValuesField.java | 9 ++++--- .../index/mapper/CustomTermFreqField.java | 10 ++++--- .../index/mapper/KeywordFieldMapper.java | 15 ++++++----- .../elasticsearch/index/mapper/Mapper.java | 27 +++++++++++++++++++ .../index/mapper/SeqNoFieldMapper.java | 8 +++--- .../index/mapper/SourceFieldMapper.java | 11 ++++---- .../index/mapper/TextFieldMapper.java | 19 ++++++------- .../index/mapper/TextSearchInfo.java | 11 ++++---- .../VersionStringFieldMapper.java | 13 ++++----- .../wildcard/mapper/WildcardFieldMapper.java | 16 ++++++----- 17 files changed, 122 insertions(+), 77 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 3b54668a91423..ad49b305c8ef7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -76,15 +76,16 @@ public class MatchOnlyTextFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "match_only_text"; public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(true); - FIELD_TYPE.setStored(false); - FIELD_TYPE.setStoreTermVectors(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.freeze(); + final FieldType ft = new FieldType(); + ft.setTokenized(true); + ft.setStored(false); + ft.setStoreTermVectors(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } } @@ -367,7 +368,7 @@ private MatchOnlyTextFieldMapper( super(simpleName, mappedFieldType, multiFields, copyTo, false, null); assert mappedFieldType.getTextSearchInfo().isTokenized(); assert mappedFieldType.hasDocValues() == false; - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.indexCreatedVersion = builder.indexCreatedVersion; this.indexAnalyzers = builder.analyzers.indexAnalyzers; this.indexAnalyzer = builder.analyzers.getIndexAnalyzer(); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index b021f5b8e6954..7342a432dd5df 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -43,13 +43,14 @@ public class RankFeatureFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "rank_feature"; public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setIndexOptions(IndexOptions.NONE); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setIndexOptions(IndexOptions.NONE); + ft.setOmitNorms(true); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java index bb8b7ad382044..ca8231c46736f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java @@ -312,7 +312,7 @@ static class SearchAsYouTypeFieldType extends StringFieldType { new TextSearchInfo(fieldType, similarity, searchAnalyzer, searchQuoteAnalyzer), meta ); - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); } public void setPrefixField(PrefixFieldType prefixField) { @@ -499,7 +499,7 @@ static final class PrefixFieldMapper extends FieldMapper { PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType) { super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); - this.fieldType = fieldType; + this.fieldType = Mapper.freezeAndDeduplicateFieldType(fieldType); } @Override @@ -538,7 +538,7 @@ static final class ShingleFieldMapper extends FieldMapper { ShingleFieldMapper(FieldType fieldType, ShingleFieldType mappedFieldtype) { super(mappedFieldtype.name(), mappedFieldtype, MultiFields.empty(), CopyTo.empty()); - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); } FieldType getLuceneFieldType() { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index d0c20e6aceabf..19f1d0455630d 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -481,7 +481,7 @@ protected ICUCollationKeywordFieldMapper( ) { super(simpleName, mappedFieldType, multiFields, copyTo, false, null); assert collator.isFrozen(); - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.params = builder.collatorParams(); this.ignoreAbove = builder.ignoreAbove.getValue(); this.collator = collator; diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 2cd46ee170c08..22de0d03de15a 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -508,7 +508,7 @@ protected AnnotatedTextFieldMapper( ) { super(simpleName, mappedFieldType, multiFields, copyTo); assert fieldType.tokenized(); - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.builder = builder; this.indexAnalyzer = wrapAnalyzer(builder.analyzers.getIndexAnalyzer()); } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 97bed0c5e06a3..706fb057cc8ee 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -37,10 +37,11 @@ public class Murmur3FieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "murmur3"; public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setIndexOptions(IndexOptions.NONE); - FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setIndexOptions(IndexOptions.NONE); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 7698c361f55d3..581c87d224f2d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -85,14 +85,15 @@ public FieldMapper.Builder getMergeBuilder() { } public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(true); - FIELD_TYPE.setStored(false); - FIELD_TYPE.setStoreTermVectors(false); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); + final FieldType ft = new FieldType(); + ft.setTokenized(true); + ft.setStored(false); + ft.setStoreTermVectors(false); + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + ft.setOmitNorms(true); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } public static final boolean DEFAULT_PRESERVE_SEPARATORS = true; public static final boolean DEFAULT_POSITION_INCREMENTS = true; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java b/server/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java index b4ce98dcdaf9a..115b20d7d88b3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java @@ -22,11 +22,12 @@ // used for binary, geo and range fields public abstract class CustomDocValuesField implements IndexableField { - public static final FieldType TYPE = new FieldType(); + public static final FieldType TYPE; static { - TYPE.setDocValuesType(DocValuesType.BINARY); - TYPE.setOmitNorms(true); - TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setDocValuesType(DocValuesType.BINARY); + ft.setOmitNorms(true); + TYPE = Mapper.freezeAndDeduplicateFieldType(ft); } private final String name; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CustomTermFreqField.java b/server/src/main/java/org/elasticsearch/index/mapper/CustomTermFreqField.java index 6ee379e598578..3b618989a2053 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CustomTermFreqField.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CustomTermFreqField.java @@ -21,11 +21,13 @@ */ public final class CustomTermFreqField extends Field { - private static final FieldType FIELD_TYPE = new FieldType(); + private static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + final FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS); + FIELD_TYPE = Mapper.freezeAndDeduplicateFieldType(ft); } private final int fieldValue; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e68e433353c83..8d9c77f503ab9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -87,14 +87,15 @@ public final class KeywordFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "keyword"; public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.setDocValuesType(DocValuesType.SORTED_SET); - FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.DOCS); + ft.setDocValuesType(DocValuesType.SORTED_SET); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } public static TextSearchInfo TEXT_SEARCH_INFO = new TextSearchInfo( @@ -831,7 +832,7 @@ private KeywordFieldMapper( this.indexed = builder.indexed.getValue(); this.hasDocValues = builder.hasDocValues.getValue(); this.indexOptions = builder.indexOptions.getValue(); - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.similarity = builder.similarity.getValue(); this.normalizerName = builder.normalizer.getValue(); this.splitQueriesOnWhitespace = builder.splitQueriesOnWhitespace.getValue(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 32796aed2f422..598e8c4d394e8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.FieldType; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.StringLiteralDeduplicator; import org.elasticsearch.index.IndexVersion; @@ -15,8 +16,11 @@ import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; public abstract class Mapper implements ToXContentFragment, Iterable { + public abstract static class Builder { protected final String name; @@ -105,4 +109,27 @@ public String toString() { public static String internFieldName(String fieldName) { return fieldNameStringDeduplicator.deduplicate(fieldName); } + + private static final Map fieldTypeDeduplicator = new ConcurrentHashMap<>(); + + /** + * Freezes the given {@link FieldType} instances and tries to deduplicate it as long as the field does not return a non-empty value for + * {@link FieldType#getAttributes()}. + * + * @param fieldType field type to deduplicate + * @return deduplicated field type + */ + public static FieldType freezeAndDeduplicateFieldType(FieldType fieldType) { + fieldType.freeze(); + var attributes = fieldType.getAttributes(); + if ((attributes != null && attributes.isEmpty() == false) || fieldType.getClass() != FieldType.class) { + // don't deduplicate subclasses or types with non-empty attribute maps to avoid memory leaks + return fieldType; + } + if (fieldTypeDeduplicator.size() > 1000) { + // guard against the case where we run up too many combinations via (vector-)dimensions combinations + fieldTypeDeduplicator.clear(); + } + return fieldTypeDeduplicator.computeIfAbsent(fieldType, Function.identity()); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 1dc1d236d80d0..8e8c58b35c68a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -50,10 +50,12 @@ public class SeqNoFieldMapper extends MetadataFieldMapper { // Like Lucene's LongField but single-valued (NUMERIC doc values instead of SORTED_NUMERIC doc values) private static class SingleValueLongField extends Field { - private static final FieldType FIELD_TYPE = new FieldType(); + private static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setDimensions(1, Long.BYTES); - FIELD_TYPE.setDocValuesType(DocValuesType.NUMERIC); + FieldType ft = new FieldType(); + ft.setDimensions(1, Long.BYTES); + ft.setDocValuesType(DocValuesType.NUMERIC); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } private final BytesRef pointValue; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index ee709be35bc82..c5d5dbec1ef15 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -77,13 +77,14 @@ private enum Mode { public static class Defaults { public static final String NAME = SourceFieldMapper.NAME; - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setIndexOptions(IndexOptions.NONE); // not indexed - FIELD_TYPE.setStored(true); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setIndexOptions(IndexOptions.NONE); // not indexed + ft.setStored(true); + ft.setOmitNorms(true); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 384a35ca4156e..4c3e1d38b6d57 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -99,15 +99,16 @@ public static class Defaults { public static final int INDEX_PREFIX_MIN_CHARS = 2; public static final int INDEX_PREFIX_MAX_CHARS = 5; - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(true); - FIELD_TYPE.setStored(false); - FIELD_TYPE.setStoreTermVectors(false); - FIELD_TYPE.setOmitNorms(false); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); - FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setTokenized(true); + ft.setStored(false); + ft.setStoreTermVectors(false); + ft.setOmitNorms(false); + ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } /** @@ -635,7 +636,7 @@ private static final class SubFieldInfo { private final String field; SubFieldInfo(String field, FieldType fieldType, Analyzer analyzer) { - this.fieldType = fieldType; + this.fieldType = Mapper.freezeAndDeduplicateFieldType(fieldType); this.analyzer = analyzer; this.field = field; } @@ -1138,7 +1139,7 @@ protected TextFieldMapper( if (fieldType.indexOptions() == IndexOptions.NONE && fieldType().fielddata()) { throw new IllegalArgumentException("Cannot enable fielddata on a [text] field that is not indexed: [" + name() + "]"); } - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.prefixFieldInfo = prefixFieldInfo; this.phraseFieldInfo = phraseFieldInfo; this.indexCreatedVersion = builder.indexCreatedVersion; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java b/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java index 744ebc6d6013a..e0cb747f48ceb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextSearchInfo.java @@ -28,12 +28,13 @@ public record TextSearchInfo( NamedAnalyzer searchQuoteAnalyzer ) { - private static final FieldType SIMPLE_MATCH_ONLY_FIELD_TYPE = new FieldType(); + private static final FieldType SIMPLE_MATCH_ONLY_FIELD_TYPE; static { - SIMPLE_MATCH_ONLY_FIELD_TYPE.setTokenized(false); - SIMPLE_MATCH_ONLY_FIELD_TYPE.setOmitNorms(true); - SIMPLE_MATCH_ONLY_FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(true); + SIMPLE_MATCH_ONLY_FIELD_TYPE = Mapper.freezeAndDeduplicateFieldType(ft); } /** @@ -100,7 +101,7 @@ public TextSearchInfo( NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer ) { - this.luceneFieldType = luceneFieldType; + this.luceneFieldType = Mapper.freezeAndDeduplicateFieldType(luceneFieldType); this.similarity = similarity; this.searchAnalyzer = Objects.requireNonNull(searchAnalyzer); this.searchQuoteAnalyzer = Objects.requireNonNull(searchQuoteAnalyzer); diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index e6dc15550f418..9e88c516576c2 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -86,13 +86,14 @@ public class VersionStringFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "version"; public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static final FieldType FIELD_TYPE; static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.freeze(); + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } } @@ -355,7 +356,7 @@ private VersionStringFieldMapper( CopyTo copyTo ) { super(simpleName, mappedFieldType, multiFields, copyTo); - this.fieldType = fieldType; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); } @Override diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index e336d1031b292..e38ee029bd123 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -879,9 +879,17 @@ protected String parseSourceValue(Object value) { } + private static final FieldType NGRAM_FIELD_TYPE; + + static { + FieldType ft = new FieldType(Defaults.FIELD_TYPE); + ft.setTokenized(true); + NGRAM_FIELD_TYPE = freezeAndDeduplicateFieldType(ft); + assert NGRAM_FIELD_TYPE.indexOptions() == IndexOptions.DOCS; + } + private final int ignoreAbove; private final String nullValue; - private final FieldType ngramFieldType; private final IndexVersion indexVersionCreated; private final boolean storeIgnored; @@ -900,10 +908,6 @@ private WildcardFieldMapper( this.ignoreAbove = ignoreAbove; this.storeIgnored = storeIgnored; this.indexVersionCreated = indexVersionCreated; - this.ngramFieldType = new FieldType(Defaults.FIELD_TYPE); - this.ngramFieldType.setTokenized(true); - this.ngramFieldType.freeze(); - assert ngramFieldType.indexOptions() == IndexOptions.DOCS; } @Override @@ -954,7 +958,7 @@ private String originalName() { void createFields(String value, LuceneDocument parseDoc, List fields) { String ngramValue = addLineEndChars(value); - Field ngramField = new Field(fieldType().name(), ngramValue, ngramFieldType); + Field ngramField = new Field(fieldType().name(), ngramValue, NGRAM_FIELD_TYPE); fields.add(ngramField); CustomBinaryDocValuesField dvField = (CustomBinaryDocValuesField) parseDoc.getByKey(fieldType().name()); From df2686b34ce71dd1b96c8abdf8ce6f299491ee8e Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 8 Sep 2023 13:25:39 -0700 Subject: [PATCH 53/69] Remove driver runner without timeout (#99386) Currently, we have the method `DriverRunner#runToCompletion`, which is exclusively used in tests. This method calls `Future.actionGet()` without a timeout, potentially causing tests to hang indefinitely if the future is never completed (see #99347). This PR removes the `DriverRunner#runToCompletion` method entirely and replaces its usages with DriverRunners that include a timeout. Relates #99347 --- .../compute/operator/DriverRunner.java | 32 ------------------- .../compute/operator/OperatorTestCase.java | 13 +++++++- .../xpack/esql/lookup/EnrichLookupIT.java | 26 +++++++++++---- .../elasticsearch/xpack/esql/CsvTests.java | 20 +++++++++--- 4 files changed, 47 insertions(+), 44 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverRunner.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverRunner.java index 9ab40b15e4623..c687ce7f864f1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverRunner.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverRunner.java @@ -9,14 +9,11 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.core.Releasables; import org.elasticsearch.tasks.TaskCancelledException; -import org.elasticsearch.threadpool.ThreadPool; import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicReference; /** @@ -90,33 +87,4 @@ private void done() { start(driver, driverListener); } } - - /** - * Run all the of the listed drivers in the supplier {@linkplain ThreadPool}. - * @return the headers added to the context while running the drivers - */ - public static Map> runToCompletion(ThreadPool threadPool, int maxIterations, List drivers) { - DriverRunner runner = new DriverRunner() { - @Override - protected void start(Driver driver, ActionListener driverListener) { - Driver.start(threadPool.executor("esql"), driver, maxIterations, driverListener); - } - }; - AtomicReference>> responseHeaders = new AtomicReference<>(); - PlainActionFuture future = new PlainActionFuture<>(); - runner.runToCompletion(drivers, new ActionListener<>() { - @Override - public void onResponse(Void unused) { - responseHeaders.set(threadPool.getThreadContext().getResponseHeaders()); - future.onResponse(null); - } - - @Override - public void onFailure(Exception e) { - future.onFailure(e); - } - }); - future.actionGet(); - return responseHeaders.get(); - } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java index f32ef67ab766c..0f47b5c29cc3b 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java @@ -7,6 +7,8 @@ package org.elasticsearch.compute.operator; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.settings.Settings; @@ -17,6 +19,7 @@ import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.TestThreadPool; @@ -178,8 +181,16 @@ public static void runDriver(List drivers) { getTestClass().getSimpleName(), new FixedExecutorBuilder(Settings.EMPTY, "esql", numThreads, 1024, "esql", EsExecutors.TaskTrackingConfig.DEFAULT) ); + var driverRunner = new DriverRunner() { + @Override + protected void start(Driver driver, ActionListener driverListener) { + Driver.start(threadPool.executor("esql"), driver, between(1, 10000), driverListener); + } + }; + PlainActionFuture future = new PlainActionFuture<>(); try { - DriverRunner.runToCompletion(threadPool, between(1, 10000), drivers); + driverRunner.runToCompletion(drivers, future); + future.actionGet(TimeValue.timeValueSeconds(30)); } finally { terminate(threadPool); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java index 8f15c6b07f5e3..86d082f1051ab 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/lookup/EnrichLookupIT.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.esql.lookup; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; @@ -22,12 +24,14 @@ import org.elasticsearch.compute.operator.DriverRunner; import org.elasticsearch.compute.operator.OutputOperator; import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.enrich.EnrichLookupOperator; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.esql.plugin.TransportEsqlQueryAction; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.NamedExpression; @@ -38,6 +42,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -84,7 +89,7 @@ public void testSimple() { ); DiscoveryNode clientNode = randomFrom(clusterService().state().nodes().stream().toList()); - var lookupService = internalCluster().getInstance(TransportEsqlQueryAction.class, clientNode.getName()).enrichLookupService(); + TransportEsqlQueryAction queryAction = internalCluster().getInstance(TransportEsqlQueryAction.class, clientNode.getName()); TransportService transportService = internalCluster().getInstance(TransportService.class, clientNode.getName()); EsqlQueryRequest parentRequest = new EsqlQueryRequest(); @@ -95,7 +100,7 @@ public void testSimple() { parentTask, randomIntBetween(1, 3), 0, - lookupService, + queryAction.enrichLookupService(), "users", "match", "uid", @@ -132,11 +137,18 @@ public void testSimple() { DateFormatter dateFmt = DateFormatter.forPattern("yyyy-MM-dd"); - DriverRunner.runToCompletion( - internalCluster().getInstance(TransportService.class).getThreadPool(), - between(1, 10_000), - List.of(new Driver(new DriverContext(), sourceOperator, List.of(enrichOperator), outputOperator, () -> {})) - ); + var runner = new DriverRunner() { + final Executor executor = transportService.getThreadPool().executor(EsqlPlugin.ESQL_THREAD_POOL_NAME); + + @Override + protected void start(Driver driver, ActionListener listener) { + Driver.start(executor, driver, between(1, 1000), listener); + } + }; + Driver driver = new Driver(new DriverContext(), sourceOperator, List.of(enrichOperator), outputOperator, () -> {}); + PlainActionFuture future = new PlainActionFuture<>(); + runner.runToCompletion(List.of(driver), future); + future.actionGet(TimeValue.timeValueSeconds(30)); transportService.getTaskManager().unregister(parentTask); Page output = outputPage.get(); assertThat(output.getBlockCount(), equalTo(4)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index fa14501edc50d..8a4a53f7b986d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -9,6 +9,8 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.settings.Settings; @@ -16,9 +18,11 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.Driver; +import org.elasticsearch.compute.operator.DriverRunner; import org.elasticsearch.compute.operator.exchange.ExchangeSinkHandler; import org.elasticsearch.compute.operator.exchange.ExchangeSourceHandler; import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -89,7 +93,6 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.compute.operator.DriverRunner.runToCompletion; import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.xpack.esql.CsvTestUtils.ExpectedResults; @@ -357,7 +360,6 @@ private ActualResults executePlan() throws Exception { List drivers = new ArrayList<>(); List collectedPages = Collections.synchronizedList(new ArrayList<>()); - Map> responseHeaders; // replace fragment inside the coordinator plan try { @@ -375,11 +377,21 @@ private ActualResults executePlan() throws Exception { drivers.addAll(dataNodeExecutionPlan.createDrivers(sessionId)); Randomness.shuffle(drivers); } - responseHeaders = runToCompletion(threadPool, between(1, 10_000), drivers); + // Execute the driver + DriverRunner runner = new DriverRunner() { + @Override + protected void start(Driver driver, ActionListener driverListener) { + Driver.start(threadPool.executor(ESQL_THREAD_POOL_NAME), driver, between(1, 1000), driverListener); + } + }; + PlainActionFuture future = new PlainActionFuture<>(); + runner.runToCompletion(drivers, future); + future.actionGet(TimeValue.timeValueSeconds(30)); + var responseHeaders = threadPool.getThreadContext().getResponseHeaders(); + return new ActualResults(columnNames, columnTypes, dataTypes, collectedPages, responseHeaders); } finally { Releasables.close(() -> Releasables.close(drivers), exchangeSource::decRef); } - return new ActualResults(columnNames, columnTypes, dataTypes, collectedPages, responseHeaders); } private Throwable reworkException(Throwable th) { From f69c5b146006aad99564809f7871c7637551ff49 Mon Sep 17 00:00:00 2001 From: Dianna Hohensee Date: Fri, 8 Sep 2023 17:12:40 -0400 Subject: [PATCH 54/69] Call TransportService#registerRequestHandler with Executor (#99385) Remove temporary TransportService#registerRequestHandler functions that take String instead of Executor and update the callers. This is part of a larger change to pass Executor into request handlers, rather than creating and stashing Strings that are only used to eventually fetch appropriate Executors anyway. Extensive testing changes were required to make TransportService and ThreadPool mocks capable of looking up Executors from Strings for TransportAction constructors. Added a new MockUtils class to modularize the additional test setup. Part of the changes for #97879 --- .../common/GrokProcessorGetActionTests.java | 7 +- .../TransportRankEvalActionTests.java | 4 +- .../reindex/UpdateByQueryWithScriptTests.java | 5 + ...rossClusterSearchUnavailableClusterIT.java | 54 ++- .../flush/TransportShardFlushAction.java | 2 +- .../TransportFieldCapabilitiesAction.java | 2 +- .../action/search/SearchTransportService.java | 25 +- .../TransportOpenPointInTimeAction.java | 3 +- .../support/HandledTransportAction.java | 9 +- .../broadcast/TransportBroadcastAction.java | 2 +- .../node/TransportBroadcastByNodeAction.java | 2 +- .../TransportBroadcastUnpromotableAction.java | 7 +- .../support/nodes/TransportNodesAction.java | 2 +- .../TransportReplicationAction.java | 12 +- ...ransportInstanceSingleOperationAction.java | 4 +- .../shard/TransportSingleShardAction.java | 10 +- .../support/tasks/TransportTasksAction.java | 7 +- .../action/shard/ShardStateAction.java | 5 +- .../cluster/coordination/Coordinator.java | 2 +- .../coordination/FollowersChecker.java | 2 +- .../cluster/coordination/JoinHelper.java | 7 +- .../coordination/JoinValidationService.java | 2 +- .../cluster/coordination/LeaderChecker.java | 2 +- .../PublicationTransportHandler.java | 2 +- .../StatefulPreVoteCollector.java | 2 +- .../elasticsearch/discovery/PeerFinder.java | 2 +- .../gateway/LocalAllocateDangledIndices.java | 3 +- .../recovery/PeerRecoverySourceService.java | 4 +- .../recovery/PeerRecoveryTargetService.java | 16 +- .../indices/store/IndicesStore.java | 3 +- .../VerifyNodeRepositoryAction.java | 2 +- .../tasks/TaskCancellationService.java | 5 +- .../transport/RemoteClusterService.java | 3 +- .../transport/TransportActionProxy.java | 3 +- .../transport/TransportService.java | 40 -- ...nsportDeleteDesiredBalanceActionTests.java | 16 +- ...TransportGetDesiredBalanceActionTests.java | 39 +- .../ClusterFormationInfoActionTests.java | 6 + .../MasterHistoryActionTests.java | 6 + ...ransportUpdateDesiredNodesActionTests.java | 20 +- .../ReservedRepositoryActionTests.java | 7 +- .../settings/ClusterGetSettingsTests.java | 7 +- .../ClusterUpdateSettingsRequestTests.java | 7 +- .../TransportCreateIndexActionTests.java | 8 +- .../TransportRolloverActionTests.java | 6 + .../TransportUpdateSettingsActionTests.java | 8 +- ...vedComposableIndexTemplateActionTests.java | 25 +- ...ActionIndicesThatCannotBeCreatedTests.java | 10 +- .../bulk/TransportBulkActionIngestTests.java | 5 + ...StateAwareHandledTransportActionTests.java | 6 +- .../replication/PostWriteRefreshTests.java | 4 +- .../TransportWriteActionTests.java | 2 +- .../PeerRecoverySourceServiceTests.java | 4 +- .../tasks/BanFailureLoggingTests.java | 2 +- .../RemoteClusterConnectionTests.java | 78 ++-- .../SniffConnectionStrategyTests.java | 5 +- .../transport/TransportActionProxyTests.java | 99 +++-- ...ortServiceDeserializationFailureTests.java | 3 +- .../TransportServiceLifecycleTests.java | 2 +- .../org/elasticsearch/test/MockUtils.java | 71 +++ .../AbstractSimpleTransportTestCase.java | 410 +++++++++++------- .../DisruptableMockTransportTests.java | 7 +- .../AnalyticsInfoTransportActionTests.java | 17 +- .../TransportAnalyticsStatsActionTests.java | 6 + ...ortDeleteAutoscalingPolicyActionTests.java | 13 +- ...nsportGetAutoscalingPolicyActionTests.java | 13 +- ...nsportPutAutoscalingPolicyActionTests.java | 13 +- .../ccr/CCRInfoTransportActionTests.java | 18 +- .../ClearCcrRestoreSessionActionTests.java | 3 + .../GetCcrRestoreFileChunkActionTests.java | 3 + .../action/TransportTermsEnumAction.java | 2 +- .../action/TransportXPackInfoActionTests.java | 4 +- ...nsportNodeDeprecationCheckActionTests.java | 7 + .../InternalExecutePolicyActionTests.java | 4 +- .../eql/EqlInfoTransportActionTests.java | 13 +- .../compute/operator/DriverTaskRunner.java | 4 +- .../operator/exchange/ExchangeService.java | 4 +- .../esql/enrich/EnrichLookupService.java | 7 +- .../xpack/esql/plugin/ComputeService.java | 9 +- .../esql/EsqlInfoTransportActionTests.java | 12 +- .../graph/GraphInfoTransportActionTests.java | 12 +- ...portPutSamlServiceProviderActionTests.java | 5 +- ...ndexLifecycleInfoTransportActionTests.java | 23 +- .../TransportDeleteLifecycleActionTests.java | 7 +- .../TransportPutLifecycleActionTests.java | 7 +- .../action/TransportStopILMActionTests.java | 7 +- .../TransportDeletePipelineActionTests.java | 4 +- .../TransportGetPipelineActionTests.java | 14 +- ...chineLearningInfoTransportActionTests.java | 14 +- .../action/TransportCloseJobActionTests.java | 7 +- ...TransportDeleteExpiredDataActionTests.java | 5 + ...nsportFinalizeJobExecutionActionTests.java | 6 +- .../MonitoringInfoTransportActionTests.java | 19 +- .../TransportMonitoringBulkActionTests.java | 5 + .../action/TransportRollupSearchAction.java | 4 +- .../RollupInfoTransportActionTests.java | 17 +- ...CrossClusterAccessHeadersForCcsRestIT.java | 60 +-- .../SecurityInfoTransportActionTests.java | 23 +- ...rtCreateCrossClusterApiKeyActionTests.java | 9 +- .../TransportGrantApiKeyActionTests.java | 8 +- ...rtUpdateCrossClusterApiKeyActionTests.java | 7 +- ...nsportProfileHasPrivilegesActionTests.java | 5 + .../TransportClearRealmCacheActionTests.java | 8 +- ...tCreateServiceAccountTokenActionTests.java | 5 +- ...tDeleteServiceAccountTokenActionTests.java | 4 +- ...TransportGetServiceAccountActionTests.java | 4 +- ...tServiceAccountCredentialsActionTests.java | 5 +- .../TransportCreateTokenActionTests.java | 16 +- .../TransportInvalidateTokenActionTests.java | 10 +- .../TransportHasPrivilegesActionTests.java | 9 +- ...ty4ServerTransportAuthenticationTests.java | 3 +- ...ransportDeleteShutdownNodeActionTests.java | 4 +- .../TransportPutShutdownNodeActionTests.java | 4 +- ...vedSnapshotLifecycleStateServiceTests.java | 13 +- .../action/TransportStopSLMActionTests.java | 8 +- .../SpatialInfoTransportActionTests.java | 17 +- .../SpatialStatsTransportActionTests.java | 3 +- .../sql/SqlInfoTransportActionTests.java | 14 +- .../TransformInfoTransportActionTests.java | 7 +- .../WatcherInfoTransportActionTests.java | 14 +- .../actions/TransportAckWatchActionTests.java | 3 +- .../actions/TransportPutWatchActionTests.java | 3 +- .../TransportWatcherStatsActionTests.java | 3 +- 123 files changed, 1074 insertions(+), 628 deletions(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/test/MockUtils.java diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java index c2110a14db7a6..09801eeda6437 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.grok.PatternBank; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -60,8 +61,9 @@ public void testResponseSerialization() throws Exception { public void testResponseSorting() { List sortedKeys = new ArrayList<>(LEGACY_TEST_PATTERNS.bank().keySet()); Collections.sort(sortedKeys); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GrokProcessorGetAction.TransportAction transportAction = new GrokProcessorGetAction.TransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), LEGACY_TEST_PATTERNS, ECS_TEST_PATTERNS @@ -109,8 +111,9 @@ public void onFailure(Exception e) { public void testEcsCompatibilityMode() { List sortedKeys = new ArrayList<>(ECS_TEST_PATTERNS.bank().keySet()); Collections.sort(sortedKeys); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GrokProcessorGetAction.TransportAction transportAction = new GrokProcessorGetAction.TransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), LEGACY_TEST_PATTERNS, ECS_TEST_PATTERNS diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java index 1b12f45ac8b51..3838916c15c64 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TransportRankEvalActionTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -74,10 +75,11 @@ public void multiSearch(MultiSearchRequest request, ActionListener { channel.sendResponse(new SearchShardsResponse(List.of(), List.of(), Collections.emptyMap())); } ); - newService.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { - InternalSearchResponse response = new InternalSearchResponse( - new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - null, - false, - null, - 1 - ); - SearchResponse searchResponse = new SearchResponse( - response, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); - channel.sendResponse(searchResponse); - }); + newService.registerRequestHandler( + SearchAction.NAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SearchRequest::new, + (request, channel, task) -> { + InternalSearchResponse response = new InternalSearchResponse( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), + InternalAggregations.EMPTY, + null, + null, + false, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + response, + null, + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + channel.sendResponse(searchResponse); + } + ); newService.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java index 0bd4458056c20..fbcfe415039ff 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java @@ -62,7 +62,7 @@ public TransportShardFlushAction( ); transportService.registerRequestHandler( PRE_SYNCED_FLUSH_ACTION_NAME, - ThreadPool.Names.FLUSH, + threadPool.executor(ThreadPool.Names.FLUSH), PreShardSyncedFlushRequest::new, new PreSyncedFlushTransportHandler(indicesService) ); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 79b9fa5099467..1dde340e0f527 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -92,7 +92,7 @@ public TransportFieldCapabilitiesAction( this.indicesService = indicesService; transportService.registerRequestHandler( ACTION_NODE_NAME, - ThreadPool.Names.SEARCH_COORDINATION, + this.searchCoordinationExecutor, FieldCapabilitiesNodeRequest::new, new NodeTransportHandler() ); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java index 9714fc7574418..d02958567a873 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.search.CanMatchShardResponse; import org.elasticsearch.search.SearchPhaseResult; @@ -450,7 +451,7 @@ public void writeTo(StreamOutput out) throws IOException { public static void registerRequestHandler(TransportService transportService, SearchService searchService) { transportService.registerRequestHandler( FREE_CONTEXT_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ScrollFreeContextRequest::new, (request, channel, task) -> { boolean freed = searchService.freeReaderContext(request.id()); @@ -460,7 +461,7 @@ public static void registerRequestHandler(TransportService transportService, Sea TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_SCROLL_ACTION_NAME, false, SearchFreeContextResponse::new); transportService.registerRequestHandler( FREE_CONTEXT_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, SearchFreeContextRequest::new, (request, channel, task) -> { boolean freed = searchService.freeReaderContext(request.id()); @@ -470,7 +471,7 @@ public static void registerRequestHandler(TransportService transportService, Sea TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, false, SearchFreeContextResponse::new); transportService.registerRequestHandler( CLEAR_SCROLL_CONTEXTS_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, TransportRequest.Empty::new, (request, channel, task) -> { searchService.freeAllScrollContexts(); @@ -486,7 +487,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( DFS_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardSearchRequest::new, (request, channel, task) -> searchService.executeDfsPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)) ); @@ -495,7 +496,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardSearchRequest::new, (request, channel, task) -> searchService.executeQueryPhase( request, @@ -512,7 +513,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_ID_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, QuerySearchRequest::new, (request, channel, task) -> { searchService.executeQueryPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -522,7 +523,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, InternalScrollSearchRequest::new, (request, channel, task) -> { searchService.executeQueryPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -532,7 +533,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_FETCH_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, InternalScrollSearchRequest::new, (request, channel, task) -> { searchService.executeFetchPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -542,7 +543,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( FETCH_ID_SCROLL_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardFetchRequest::new, (request, channel, task) -> { searchService.executeFetchPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel)); @@ -552,7 +553,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( FETCH_ID_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, true, true, ShardFetchSearchRequest::new, @@ -565,7 +566,7 @@ public static void registerRequestHandler(TransportService transportService, Sea // this is cheap, it does not fetch during the rewrite phase, so we can let it quickly execute on a networking thread transportService.registerRequestHandler( QUERY_CAN_MATCH_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardSearchRequest::new, (request, channel, task) -> { searchService.canMatch(request, new ChannelActionListener<>(channel)); @@ -575,7 +576,7 @@ public static void registerRequestHandler(TransportService transportService, Sea transportService.registerRequestHandler( QUERY_CAN_MATCH_NODE_NAME, - ThreadPool.Names.SEARCH_COORDINATION, + transportService.getThreadPool().executor(ThreadPool.Names.SEARCH_COORDINATION), CanMatchNodeRequest::new, (request, channel, task) -> { searchService.canMatch(request, new ChannelActionListener<>(channel)); diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java index 9c78e5ad62aea..eb854be99562e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchPhaseResult; @@ -68,7 +69,7 @@ public TransportOpenPointInTimeAction( this.searchTransportService = searchTransportService; transportService.registerRequestHandler( OPEN_SHARD_READER_CONTEXT_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardOpenReaderRequest::new, new ShardOpenReaderRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java index 24668d7130d52..a0e36a538913d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java @@ -61,7 +61,14 @@ protected HandledTransportAction( String executor ) { super(actionName, actionFilters, transportService.getTaskManager()); - transportService.registerRequestHandler(actionName, executor, false, canTripCircuitBreaker, requestReader, new TransportHandler()); + transportService.registerRequestHandler( + actionName, + transportService.getThreadPool().executor(executor), + false, + canTripCircuitBreaker, + requestReader, + new TransportHandler() + ); } class TransportHandler implements TransportRequestHandler { diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java index 570514355da56..ddd2cc43005e2 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java @@ -75,7 +75,7 @@ protected TransportBroadcastAction( transportService.registerRequestHandler( transportShardAction, - executor, + this.executor, shardRequestReader, (request, channel, task) -> ActionListener.completeWith( new ChannelActionListener<>(channel), diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index 15c28bdf33c87..4f78d68daa8f2 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -119,7 +119,7 @@ public TransportBroadcastByNodeAction( transportService.registerRequestHandler( transportNodeBroadcastAction, - executor, + this.executor, false, canTripCircuitBreaker, NodeRequest::new, diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java index d5c7fc763fba6..579ed101b56ae 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java @@ -60,7 +60,12 @@ protected TransportBroadcastUnpromotableAction( this.transportUnpromotableAction = actionName + "[u]"; this.executor = transportService.getThreadPool().executor(executor); - transportService.registerRequestHandler(transportUnpromotableAction, executor, requestReader, new UnpromotableTransportHandler()); + transportService.registerRequestHandler( + transportUnpromotableAction, + this.executor, + requestReader, + new UnpromotableTransportHandler() + ); } protected abstract void unpromotableShardOperation(Task task, Request request, ActionListener listener); diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java index 1efe70bcb5adc..b45c448ecc52c 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java @@ -82,7 +82,7 @@ protected TransportNodesAction( this.transportService = Objects.requireNonNull(transportService); this.finalExecutor = threadPool.executor(executor); this.transportNodeAction = actionName + "[n]"; - transportService.registerRequestHandler(transportNodeAction, executor, nodeRequest, new NodeTransportHandler()); + transportService.registerRequestHandler(transportNodeAction, finalExecutor, nodeRequest, new NodeTransportHandler()); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 5651af11719cf..d1e2b6dd98faa 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; @@ -187,11 +188,16 @@ protected TransportReplicationAction( this.retryTimeout = REPLICATION_RETRY_TIMEOUT.get(settings); this.forceExecutionOnPrimary = forceExecutionOnPrimary; - transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, requestReader, this::handleOperationRequest); + transportService.registerRequestHandler( + actionName, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + requestReader, + this::handleOperationRequest + ); transportService.registerRequestHandler( transportPrimaryAction, - executor, + threadPool.executor(executor), forceExecutionOnPrimary, true, in -> new ConcreteShardRequest<>(requestReader, in), @@ -201,7 +207,7 @@ protected TransportReplicationAction( // we must never reject on because of thread pool capacity on replicas transportService.registerRequestHandler( transportReplicaAction, - executor, + threadPool.executor(executor), true, true, in -> new ConcreteReplicaRequest<>(replicaRequestReader, in), diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index 63ff4cef5a924..c526a2679306d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; @@ -33,7 +34,6 @@ import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.threadpool.ThreadPool.Names; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportException; @@ -72,7 +72,7 @@ protected TransportInstanceSingleOperationAction( this.transportService = transportService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.shardActionName = actionName + "[s]"; - transportService.registerRequestHandler(shardActionName, Names.SAME, request, new ShardTransportHandler()); + transportService.registerRequestHandler(shardActionName, EsExecutors.DIRECT_EXECUTOR_SERVICE, request, new ShardTransportHandler()); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java index 001a99ecd505d..9d980fabc6714 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.LoggerMessageFormat; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; @@ -79,9 +80,14 @@ protected TransportSingleShardAction( this.executor = executor; if (isSubAction() == false) { - transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, request, new TransportHandler()); + transportService.registerRequestHandler(actionName, EsExecutors.DIRECT_EXECUTOR_SERVICE, request, new TransportHandler()); } - transportService.registerRequestHandler(transportShardAction, ThreadPool.Names.SAME, request, new ShardTransportHandler()); + transportService.registerRequestHandler( + transportShardAction, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + request, + new ShardTransportHandler() + ); } /** diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index b509008af2b4f..fb29e8a838db6 100644 --- a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -79,7 +79,12 @@ protected TransportTasksAction( this.responsesReader = responsesReader; this.responseReader = responseReader; - transportService.registerRequestHandler(transportNodeAction, nodeExecutor, NodeTaskRequest::new, new NodeTransportHandler()); + transportService.registerRequestHandler( + transportNodeAction, + transportService.getThreadPool().executor(nodeExecutor), + NodeTaskRequest::new, + new NodeTransportHandler() + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index 97bf6cc01c4e5..f9aa2c6c234be 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; @@ -98,13 +99,13 @@ public ShardStateAction( transportService.registerRequestHandler( SHARD_STARTED_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, StartedShardEntry::new, new ShardStartedTransportHandler(clusterService, new ShardStartedClusterStateTaskExecutor(allocationService, rerouteService)) ); transportService.registerRequestHandler( SHARD_FAILED_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, FailedShardEntry::new, new ShardFailedTransportHandler(clusterService, new ShardFailedClusterStateTaskExecutor(allocationService, rerouteService)) ); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index dc52791a5d5e2..7ccea8e99918b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -271,7 +271,7 @@ public Coordinator( ); transportService.registerRequestHandler( COMMIT_STATE_ACTION_NAME, - Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, ApplyCommitRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java index ad2faaccf0e96..99ab8650a52d8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java @@ -124,7 +124,7 @@ public FollowersChecker( updateFastResponseState(0, Mode.CANDIDATE); transportService.registerRequestHandler( FOLLOWER_CHECK_ACTION_NAME, - Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, FollowerCheckRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java index ce2754fa3854c..247034c88ed62 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; @@ -114,7 +115,7 @@ public class JoinHelper { transportService.registerRequestHandler( JOIN_ACTION_NAME, - Names.CLUSTER_COORDINATION, + transportService.getThreadPool().executor(Names.CLUSTER_COORDINATION), false, false, JoinRequest::new, @@ -126,7 +127,7 @@ public class JoinHelper { transportService.registerRequestHandler( START_JOIN_ACTION_NAME, - Names.CLUSTER_COORDINATION, + transportService.getThreadPool().executor(Names.CLUSTER_COORDINATION), false, false, StartJoinRequest::new, @@ -139,7 +140,7 @@ public class JoinHelper { transportService.registerRequestHandler( JOIN_PING_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, TransportRequest.Empty::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java index 248137f03fcaa..d9911ad12df84 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java @@ -121,7 +121,7 @@ public JoinValidationService( final var dataPaths = Environment.PATH_DATA_SETTING.get(settings); transportService.registerRequestHandler( JoinValidationService.JOIN_VALIDATE_ACTION_NAME, - ThreadPool.Names.CLUSTER_COORDINATION, + this.responseExecutor, ValidateJoinRequest::new, (request, channel, task) -> { final var remoteState = request.getOrReadState(); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java index 8a20e8e56d751..9fcae5bcf67f8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java @@ -113,7 +113,7 @@ public class LeaderChecker { transportService.registerRequestHandler( LEADER_CHECK_ACTION_NAME, - Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, LeaderCheckRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java index 45079b2bccd60..781a05d535b16 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java @@ -106,7 +106,7 @@ public PublicationTransportHandler( transportService.registerRequestHandler( PUBLISH_STATE_ACTION_NAME, - ThreadPool.Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, BytesTransportRequest::new, diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java index 28ab04e5a7ccc..7bc3514206ffe 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/StatefulPreVoteCollector.java @@ -64,7 +64,7 @@ public StatefulPreVoteCollector( transportService.registerRequestHandler( REQUEST_PRE_VOTE_ACTION_NAME, - Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, PreVoteRequest::new, diff --git a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java index 046f0a1c64bb5..46e8f5210e447 100644 --- a/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java +++ b/server/src/main/java/org/elasticsearch/discovery/PeerFinder.java @@ -110,7 +110,7 @@ public PeerFinder( transportService.registerRequestHandler( REQUEST_PEERS_ACTION_NAME, - Names.CLUSTER_COORDINATION, + this.clusterCoordinationExecutor, false, false, PeersRequest::new, diff --git a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java index d0579342f5bd0..0262e37dd74a9 100644 --- a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java +++ b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java @@ -35,7 +35,6 @@ import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestHandler; @@ -73,7 +72,7 @@ public LocalAllocateDangledIndices( this.indexMetadataVerifier = indexMetadataVerifier; transportService.registerRequestHandler( ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, AllocateDangledRequest::new, new AllocateDangledRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java index 21b8ca31f01a7..eac119f920f6a 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java @@ -79,7 +79,7 @@ public PeerRecoverySourceService( // node. Upon receiving START_RECOVERY, the source node will initiate the peer recovery. transportService.registerRequestHandler( Actions.START_RECOVERY, - ThreadPool.Names.GENERIC, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), StartRecoveryRequest::new, (request, channel, task) -> recover(request, task, new ChannelActionListener<>(channel)) ); @@ -89,7 +89,7 @@ public PeerRecoverySourceService( // action will fail and the target node will send a new START_RECOVERY request. transportService.registerRequestHandler( Actions.REESTABLISH_RECOVERY, - ThreadPool.Names.GENERIC, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), ReestablishRecoveryRequest::new, (request, channel, task) -> reestablish(request, new ChannelActionListener<>(channel)) ); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java index 5f06519f84c0c..b3b23ac14d158 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java @@ -122,7 +122,7 @@ public PeerRecoveryTargetService( transportService.registerRequestHandler( Actions.FILES_INFO, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryFilesInfoRequest::new, new RecoveryRequestHandler<>() { @Override @@ -140,7 +140,7 @@ protected void handleRequest(RecoveryFilesInfoRequest request, RecoveryTarget ta ); transportService.registerRequestHandler( Actions.RESTORE_FILE_FROM_SNAPSHOT, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoverySnapshotFileRequest::new, new RecoveryRequestHandler<>() { @Override @@ -151,13 +151,13 @@ protected void handleRequest(RecoverySnapshotFileRequest request, RecoveryTarget ); transportService.registerRequestHandler( Actions.FILE_CHUNK, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryFileChunkRequest::new, new FileChunkTransportRequestHandler() ); transportService.registerRequestHandler( Actions.CLEAN_FILES, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryCleanFilesRequest::new, new RecoveryRequestHandler<>() { @Override @@ -179,7 +179,7 @@ protected void handleRequest(RecoveryCleanFilesRequest request, RecoveryTarget t ); transportService.registerRequestHandler( Actions.PREPARE_TRANSLOG, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryPrepareForTranslogOperationsRequest::new, new RecoveryRequestHandler<>() { @Override @@ -194,13 +194,13 @@ protected void handleRequest( ); transportService.registerRequestHandler( Actions.TRANSLOG_OPS, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryTranslogOperationsRequest::new, new TranslogOperationsRequestHandler() ); transportService.registerRequestHandler( Actions.FINALIZE, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryFinalizeRecoveryRequest::new, new RecoveryRequestHandler<>() { @Override @@ -215,7 +215,7 @@ protected void handleRequest( ); transportService.registerRequestHandler( Actions.HANDOFF_PRIMARY_CONTEXT, - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), RecoveryHandoffPrimaryContextRequest::new, new HandoffPrimaryContextRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index e12a38e383397..f949dd59e8968 100644 --- a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -102,7 +103,7 @@ public IndicesStore( this.threadPool = threadPool; transportService.registerRequestHandler( ACTION_SHARD_EXISTS, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ShardActiveRequest::new, new ShardActiveRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java index ad6cbbaad1cdc..d940415c38916 100644 --- a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java @@ -56,7 +56,7 @@ public VerifyNodeRepositoryAction( this.repositoriesService = repositoriesService; transportService.registerRequestHandler( ACTION_NAME, - ThreadPool.Names.SNAPSHOT, + transportService.getThreadPool().executor(ThreadPool.Names.SNAPSHOT), VerifyNodeRepositoryRequest::new, new VerifyNodeRepositoryRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java index 39a76f3508daa..6ab95072727c0 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.NodeDisconnectedException; @@ -60,13 +61,13 @@ public TaskCancellationService(TransportService transportService) { this.deduplicator = new ResultDeduplicator<>(transportService.getThreadPool().getThreadContext()); transportService.registerRequestHandler( BAN_PARENT_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, BanParentTaskRequest::new, new BanParentRequestHandler() ); transportService.registerRequestHandler( CANCEL_CHILD_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, CancelChildRequest::new, new CancelChildRequestHandler() ); diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java index 9542d4b366ded..c38f4b26c665f 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.node.ReportingService; @@ -539,7 +540,7 @@ Collection getConnections() { static void registerRemoteClusterHandshakeRequestHandler(TransportService transportService) { transportService.registerRequestHandler( REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, TransportService.HandshakeRequest::new, diff --git a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java index ed3432f66087b..aa28f8a76b58e 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -188,7 +189,7 @@ public static void registerProxyActionWithDynamicResponseType( RequestHandlerRegistry requestHandler = service.getRequestHandler(action); service.registerRequestHandler( getProxyAction(action), - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, true, false, in -> cancellable diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 83fc21396e0f6..5369b9a9eec13 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -1118,46 +1118,6 @@ public static boolean isValidActionName(String actionName) { return false; } - /** - * Temporary passthrough function that continues to take a String rather than Executor type. - * - * @param action - * @param executor - * @param requestReader - * @param handler - * @param - */ - public void registerRequestHandler( - String action, - String executor, - Writeable.Reader requestReader, - TransportRequestHandler handler - ) { - registerRequestHandler(action, threadPool.executor(executor), requestReader, handler); - } - - /** - * Temporary passthrough function that continues to take a String rather than Executor type. - * - * @param action - * @param executor - * @param forceExecution - * @param canTripCircuitBreaker - * @param requestReader - * @param handler - * @param - */ - public void registerRequestHandler( - String action, - String executor, - boolean forceExecution, - boolean canTripCircuitBreaker, - Writeable.Reader requestReader, - TransportRequestHandler handler - ) { - registerRequestHandler(action, threadPool.executor(executor), forceExecution, canTripCircuitBreaker, requestReader, handler); - } - /** * Registers a new request handler * diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java index f5e812e7d0983..a2bfa68fdf25f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.gateway.TestGatewayAllocator; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -56,17 +57,20 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportDeleteDesiredBalanceActionTests extends ESAllocationTestCase { public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception { - var listener = new PlainActionFuture(); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + new TransportDeleteDesiredBalanceAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(AllocationService.class), @@ -131,8 +135,12 @@ public DesiredBalance compute( var listener = new PlainActionFuture(); + // TODO: temporary, remove in #97879 + TransportService transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + var action = new TransportDeleteDesiredBalanceAction( - mock(TransportService.class), + transportService, clusterService, threadPool, mock(ActionFilters.class), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java index 804dbf66ac0d0..80f0b435645e6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; @@ -45,6 +46,7 @@ import org.elasticsearch.test.AbstractChunkedSerializingTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.junit.Before; import java.util.HashMap; import java.util.List; @@ -59,6 +61,7 @@ import static org.elasticsearch.cluster.ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -66,16 +69,27 @@ public class TransportGetDesiredBalanceActionTests extends ESAllocationTestCase private final DesiredBalanceShardsAllocator desiredBalanceShardsAllocator = mock(DesiredBalanceShardsAllocator.class); private final ClusterInfoService clusterInfoService = mock(ClusterInfoService.class); - private final TransportGetDesiredBalanceAction transportGetDesiredBalanceAction = new TransportGetDesiredBalanceAction( - mock(TransportService.class), - mock(ClusterService.class), - mock(ThreadPool.class), - mock(ActionFilters.class), - mock(IndexNameExpressionResolver.class), - desiredBalanceShardsAllocator, - clusterInfoService, - TEST_WRITE_LOAD_FORECASTER - ); + private TransportService transportService = mock(TransportService.class); + private ThreadPool threadPool = mock(ThreadPool.class); + private TransportGetDesiredBalanceAction transportGetDesiredBalanceAction; + + @Before + public void initialize() { + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + + transportGetDesiredBalanceAction = new TransportGetDesiredBalanceAction( + transportService, + mock(ClusterService.class), + threadPool, + mock(ActionFilters.class), + mock(IndexNameExpressionResolver.class), + desiredBalanceShardsAllocator, + clusterInfoService, + TEST_WRITE_LOAD_FORECASTER + ); + } private static DesiredBalanceResponse execute(TransportGetDesiredBalanceAction action, ClusterState clusterState) throws Exception { return PlainActionFuture.get( @@ -96,11 +110,10 @@ private DesiredBalanceResponse executeAction(ClusterState clusterState) throws E public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception { var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataWithConfiguredAllocator(BALANCED_ALLOCATOR)).build(); - final var action = new TransportGetDesiredBalanceAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(ShardsAllocator.class), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java index e560959dfacf8..f190c422e165c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.monitor.StatusInfo; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Map; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -162,6 +164,10 @@ public void testTransportDoExecute() { Coordinator coordinator = mock(Coordinator.class); ClusterFormationFailureHelper.ClusterFormationState clusterFormationState = getClusterFormationState(); when(coordinator.getClusterFormationState()).thenReturn(clusterFormationState); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); ClusterFormationInfoAction.TransportAction action = new ClusterFormationInfoAction.TransportAction( transportService, actionFilters, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java index 3674a711d8968..f362327a3d8e4 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; import org.elasticsearch.threadpool.ThreadPool; @@ -24,6 +25,7 @@ import java.util.ArrayList; import java.util.List; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -83,6 +85,10 @@ public void testTransportDoExecute() { when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis()); MasterHistory masterHistory = new MasterHistory(threadPool, clusterService); when(masterHistoryService.getLocalMasterHistory()).thenReturn(masterHistory); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); MasterHistoryAction.TransportAction action = new MasterHistoryAction.TransportAction( transportService, actionFilters, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java index e4e8293810f56..508cb9b304c59 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -54,10 +55,12 @@ public void validate(List desiredNodes) {} }; public void testWriteBlocks() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_OP_SETTINGS_VALIDATOR, @@ -79,10 +82,12 @@ public void testWriteBlocks() { } public void testNoBlocks() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_OP_SETTINGS_VALIDATOR, @@ -103,10 +108,13 @@ public void validate(List desiredNodes) { } }; ClusterService clusterService = mock(ClusterService.class); + + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportUpdateDesiredNodesAction action = new TransportUpdateDesiredNodesAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), validator, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java index 3b3a996e50cd3..1b3e0ae6fe7bf 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentParser; @@ -132,14 +133,16 @@ public Repository create(RepositoryMetadata metadata) { } }; + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); RepositoriesService repositoriesService = spy( new RepositoriesService( Settings.EMPTY, mock(ClusterService.class), - mock(TransportService.class), + transportService, Map.of(), Map.of("fs", fsFactory), - mock(ThreadPool.class), + threadPool, null ) ); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java index 9e757894bcdda..00b500254883c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -47,10 +48,12 @@ public void testRequestConstruction() { public void testTransportFilters() throws Exception { final SettingsFilter filter = new SettingsFilter(List.of("persistent.foo.filtered", "transient.foo.filtered")); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportClusterGetSettingsAction action = new TransportClusterGetSettingsAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, filter, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java index ebe77f4d3ed4c..6373b94ffb94a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -85,10 +86,12 @@ private static ClusterUpdateSettingsRequest createTestItem() { public void testOperatorHandler() throws IOException { ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportClusterUpdateSettingsAction action = new TransportClusterUpdateSettingsAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), clusterSettings diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java index 671bf13fb0e77..d68641d04dd74 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -90,10 +91,13 @@ public void setUp() throws Exception { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, SYSTEM_INDICES); this.metadataCreateIndexService = mock(MetadataCreateIndexService.class); + + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); this.action = new TransportCreateIndexAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, metadataCreateIndexService, mock(ActionFilters.class), indexNameExpressionResolver, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 12308036689c6..bb1cf85013498 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.cache.query.QueryCacheStats; @@ -78,6 +79,7 @@ import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -354,6 +356,10 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr EmptySystemIndices.INSTANCE, WriteLoadForecaster.DEFAULT ); + + // TODO: temporary, remove in #97879 + when(mockTransportService.getThreadPool()).thenReturn(mockThreadPool); + when(mockThreadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); final TransportRolloverAction transportRolloverAction = new TransportRolloverAction( mockTransportService, mockClusterService, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java index 0e1581606f803..807462b6052c3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsActionTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -68,10 +69,13 @@ public void setUp() throws Exception { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, SYSTEM_INDICES); MetadataUpdateSettingsService metadataUpdateSettingsService = mock(MetadataUpdateSettingsService.class); + + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); this.action = new TransportUpdateSettingsAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, metadataUpdateSettingsService, mock(ActionFilters.class), indexNameExpressionResolver, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java index 553dd2368e5bc..28476a0d8b839 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -673,10 +674,12 @@ public void testAddRemoveIndexTemplatesWithOverlap() throws Exception { } public void testHandlerCorrectness() { + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putIndexAction = new TransportPutComposableIndexTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null @@ -687,9 +690,9 @@ public void testHandlerCorrectness() { containsInAnyOrder(reservedComposableIndexName("aaa")) ); var delIndexAction = new TransportDeleteComposableIndexTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null @@ -701,9 +704,9 @@ public void testHandlerCorrectness() { ); var putComponentAction = new TransportPutComponentTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null, @@ -716,9 +719,9 @@ public void testHandlerCorrectness() { ); var delComponentAction = new TransportDeleteComponentTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null @@ -921,10 +924,12 @@ public void testTemplatesWithReservedPrefix() throws Exception { PutComposableIndexTemplateAction.Request pr = new PutComposableIndexTemplateAction.Request(conflictingTemplateName); + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putTemplateAction = new TransportPutComposableIndexTemplateAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, null, mock(ActionFilters.class), null diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index ff96f4a00b883..1b0c24664be31 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; @@ -32,6 +31,7 @@ import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -42,7 +42,6 @@ import java.util.function.Function; import static java.util.Collections.emptySet; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -107,9 +106,6 @@ private void indicesThatCannotBeCreatedTestCase( when(clusterService.localNode()).thenReturn(localNode); when(localNode.isIngestNode()).thenReturn(randomBoolean()); - final ThreadPool threadPool = mock(ThreadPool.class); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver( new ThreadContext(Settings.EMPTY), EmptySystemIndices.INSTANCE @@ -120,9 +116,11 @@ public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) } }; + final ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportBulkAction action = new TransportBulkAction( threadPool, - mock(TransportService.class), + transportService, clusterService, null, null, diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index fd84a1e5fe816..85a74174b094f 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -183,6 +183,11 @@ public void setupAction() { MockitoAnnotations.openMocks(this); // setup services that will be called by action transportService = mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + clusterService = mock(ClusterService.class); localIngest = true; // setup nodes for local and remote diff --git a/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java b/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java index 93c4ea98adcd3..c2ff59f0ccbe0 100644 --- a/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/ReservedStateAwareHandledTransportActionTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -47,7 +48,8 @@ public void testRejectImmutableConflictClusterStateUpdate() { ClusterService clusterService = mock(ClusterService.class); doReturn(clusterState).when(clusterService).state(); - Action handler = new Action("internal:testAction", clusterService, mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + Action handler = new Action("internal:testAction", clusterService, transportService, mock(ActionFilters.class)); // nothing should happen here, since the request doesn't touch any of the immutable state keys var future = new PlainActionFuture(); @@ -61,7 +63,7 @@ public void testRejectImmutableConflictClusterStateUpdate() { FakeReservedStateAwareAction action = new FakeReservedStateAwareAction( "internal:testClusterSettings", clusterService, - mock(TransportService.class), + transportService, mock(ActionFilters.class), null ); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java index e45fc51cf6bdd..d14429647c7d3 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/PostWriteRefreshTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.engine.DocIdSeqNoAndSource; @@ -31,7 +32,6 @@ import org.elasticsearch.index.shard.ReplicationGroup; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -67,7 +67,7 @@ public void setUp() throws Exception { transportService.acceptIncomingRequests(); transportService.registerRequestHandler( TransportUnpromotableShardRefreshAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, UnpromotableShardRefreshRequest::new, (request, channel, task) -> { unpromotableRefreshRequestReceived.set(true); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java index 8e95277bdc84f..7726feaa30868 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -424,7 +424,7 @@ protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentF ), TransportWriteActionTests.this.clusterService, null, - null, + TransportWriteActionTests.threadPool, null, new ActionFilters(new HashSet<>()), TestRequest::new, diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java index 66503bf4cea79..a18e7e8ce46f9 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.NodeRoles; import org.elasticsearch.transport.TransportService; @@ -38,8 +39,9 @@ public void testDuplicateRecoveries() throws IOException { final ClusterService clusterService = mock(ClusterService.class); when(clusterService.getSettings()).thenReturn(NodeRoles.dataNode()); when(indicesService.clusterService()).thenReturn(clusterService); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService( - mock(TransportService.class), + transportService, indicesService, new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), mock(RecoveryPlannerService.class) diff --git a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java index 4ffd1e3ed94eb..56e72c25802e3 100644 --- a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java +++ b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java @@ -132,7 +132,7 @@ private void runTest( childTransportService.getTaskManager().setTaskCancellationService(new TaskCancellationService(childTransportService)); childTransportService.registerRequestHandler( "internal:testAction[c]", - ThreadPool.Names.MANAGEMENT, // busy-wait for cancellation but not on a transport thread + threadPool.executor(ThreadPool.Names.MANAGEMENT), // busy-wait for cancellation but not on a transport thread (StreamInput in) -> new TransportRequest.Empty(in) { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 5635e07000029..2d0956fae5499 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.SuppressForbidden; @@ -125,7 +126,7 @@ public static MockTransportService startTransport( try { newService.registerRequestHandler( SearchShardsAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, SearchShardsRequest::new, (request, channel, task) -> { if ("index_not_found".equals(request.preference())) { @@ -135,41 +136,50 @@ public static MockTransportService startTransport( } } ); - newService.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { - if ("index_not_found".equals(request.preference())) { - channel.sendResponse(new IndexNotFoundException("index")); - return; - } - SearchHits searchHits; - if ("null_target".equals(request.preference())) { - searchHits = new SearchHits(new SearchHit[] { new SearchHit(0) }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1F); - } else { - searchHits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN); + newService.registerRequestHandler( + SearchAction.NAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SearchRequest::new, + (request, channel, task) -> { + if ("index_not_found".equals(request.preference())) { + channel.sendResponse(new IndexNotFoundException("index")); + return; + } + SearchHits searchHits; + if ("null_target".equals(request.preference())) { + searchHits = new SearchHits( + new SearchHit[] { new SearchHit(0) }, + new TotalHits(1, TotalHits.Relation.EQUAL_TO), + 1F + ); + } else { + searchHits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN); + } + InternalSearchResponse response = new InternalSearchResponse( + searchHits, + InternalAggregations.EMPTY, + null, + null, + false, + null, + 1 + ); + SearchResponse searchResponse = new SearchResponse( + response, + null, + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + channel.sendResponse(searchResponse); } - InternalSearchResponse response = new InternalSearchResponse( - searchHits, - InternalAggregations.EMPTY, - null, - null, - false, - null, - 1 - ); - SearchResponse searchResponse = new SearchResponse( - response, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); - channel.sendResponse(searchResponse); - }); + ); newService.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); @@ -183,7 +193,7 @@ public static MockTransportService startTransport( if (RemoteClusterPortSettings.REMOTE_CLUSTER_SERVER_ENABLED.get(s)) { newService.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(knownNodes)) ); diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java index 577f5d2c7dc18..84e4249bb0ccb 100644 --- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; @@ -120,7 +121,7 @@ public MockTransportService startTransport( try { newService.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); @@ -134,7 +135,7 @@ public MockTransportService startTransport( if (hasClusterCredentials) { newService.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(knownNodes)) ); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java index 704de4fb2276a..32a5b5dec9597 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.RefCounted; @@ -108,32 +109,47 @@ private MockTransportService buildService(VersionInformation version, TransportV } public void testSendMessage() throws InterruptedException { - serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_A"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - }); + serviceA.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_A"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + } + ); final boolean cancellable = randomBoolean(); TransportActionProxy.registerProxyAction(serviceA, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceA, nodeB); - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertThat(task instanceof CancellableTask, equalTo(cancellable)); - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_B"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - }); + serviceB.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertThat(task instanceof CancellableTask, equalTo(cancellable)); + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_B"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + } + ); TransportActionProxy.registerProxyAction(serviceB, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceB, nodeC); - serviceC.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertThat(task instanceof CancellableTask, equalTo(cancellable)); - assertEquals(request.sourceNode, "TS_A"); - final SimpleTestResponse response = new SimpleTestResponse("TS_C"); - channel.sendResponse(response); - assertThat(response.hasReferences(), equalTo(false)); - }); + serviceC.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertThat(task instanceof CancellableTask, equalTo(cancellable)); + assertEquals(request.sourceNode, "TS_A"); + final SimpleTestResponse response = new SimpleTestResponse("TS_C"); + channel.sendResponse(response); + assertThat(response.hasReferences(), equalTo(false)); + } + ); TransportActionProxy.registerProxyAction(serviceC, "internal:test", cancellable, SimpleTestResponse::new); // Node A -> Node B -> Node C: different versions - serialize the response @@ -248,7 +264,7 @@ public void testSendLocalRequest() throws Exception { final boolean cancellable = randomBoolean(); serviceB.registerRequestHandler( "internal:test", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), SimpleTestRequest::new, (request, channel, task) -> { try { @@ -308,24 +324,39 @@ public void handleException(TransportException exp) { public void testException() throws InterruptedException { boolean cancellable = randomBoolean(); - serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - SimpleTestResponse response = new SimpleTestResponse("TS_A"); - channel.sendResponse(response); - }); + serviceA.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + SimpleTestResponse response = new SimpleTestResponse("TS_A"); + channel.sendResponse(response); + } + ); TransportActionProxy.registerProxyAction(serviceA, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceA, nodeB); - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - assertEquals(request.sourceNode, "TS_A"); - SimpleTestResponse response = new SimpleTestResponse("TS_B"); - channel.sendResponse(response); - }); + serviceB.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + assertEquals(request.sourceNode, "TS_A"); + SimpleTestResponse response = new SimpleTestResponse("TS_B"); + channel.sendResponse(response); + } + ); TransportActionProxy.registerProxyAction(serviceB, "internal:test", cancellable, SimpleTestResponse::new); AbstractSimpleTransportTestCase.connectToNode(serviceB, nodeC); - serviceC.registerRequestHandler("internal:test", ThreadPool.Names.SAME, SimpleTestRequest::new, (request, channel, task) -> { - throw new ElasticsearchException("greetings from TS_C"); - }); + serviceC.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SimpleTestRequest::new, + (request, channel, task) -> { + throw new ElasticsearchException("greetings from TS_C"); + } + ); TransportActionProxy.registerProxyAction(serviceC, "internal:test", cancellable, SimpleTestResponse::new); CountDownLatch latch = new CountDownLatch(1); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java index b6d3ab5455fd6..2eb77c706a3a2 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; @@ -68,7 +69,7 @@ protected void onSendRequest(long requestId, String action, TransportRequest req transportService.registerRequestHandler( testActionName, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, TransportRequest.Empty::new, (request, channel, task) -> channel.sendResponse(TransportResponse.Empty.INSTANCE) ); diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java index 6f422190e9870..c5aa918fdc0e2 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java @@ -155,7 +155,7 @@ public ExecutorService executor(String name) { for (final var executor : EXECUTOR_NAMES) { transportService.registerRequestHandler( ACTION_NAME_PREFIX + executor, - executor, + threadPool.executor(executor), TransportRequest.Empty::new, (request, channel, task) -> { if (randomBoolean()) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java b/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java new file mode 100644 index 0000000000000..4edca754df053 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/MockUtils.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test; + +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockingDetails; +import static org.mockito.Mockito.when; + +/** + * Utilities for setting up Mockito mocks. + */ +public class MockUtils { + + /** + * Sets up a mock TransportService that can answer calls to TransportService.getThreadPool().executor(String). + * + * @return A mocked TransportService instance + */ + public static TransportService setupTransportServiceWithThreadpoolExecutor() { + TransportService transportService = mock(TransportService.class); + ThreadPool threadPool = mock(ThreadPool.class); + + setMockReturns(transportService, threadPool); + + return transportService; + } + + /** + * Sets up a mock TransportService that can answer calls to TransportService.getThreadPool().executor(String), using the given + * threadPool in TransportService. + * + * @param threadPool A mock ThreadPool + * @return A mocked TransportService instance + */ + public static TransportService setupTransportServiceWithThreadpoolExecutor(ThreadPool threadPool) { + assert mockingDetails(threadPool).isMock(); + TransportService transportService = mock(TransportService.class); + + setMockReturns(transportService, threadPool); + + return transportService; + } + + /** + * Sets up the given mock TransportService so that it can answer calls to TransportService.getThreadPool().executor(String). + * + * @param transportService A mock TransportService to be set up. + */ + public static void setupTransportServiceWithThreadpoolExecutor(TransportService transportService) { + assert mockingDetails(transportService).isMock(); + ThreadPool threadPool = mock(ThreadPool.class); + + setMockReturns(transportService, threadPool); + } + + private static void setMockReturns(TransportService transportService, ThreadPool threadPool) { + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index 4a683c3f7dd57..2a80b4e2e7e45 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -313,7 +313,7 @@ public void assertNoPendingHandshakes(Transport transport) { public void testHelloWorld() { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -397,7 +397,7 @@ public void testThreadContext() throws ExecutionException, InterruptedException serviceA.registerRequestHandler( "internal:ping_pong", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertEquals("ping_user", threadPool.getThreadContext().getHeader("test.ping.user")); @@ -460,7 +460,7 @@ public void testLocalNodeConnection() throws InterruptedException { final AtomicReference exception = new AtomicReference<>(); serviceA.registerRequestHandler( "internal:localNode", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { try { @@ -519,8 +519,8 @@ public void testMessageListeners() throws Exception { } }; final String ACTION = "internal:action"; - serviceA.registerRequestHandler(ACTION, ThreadPool.Names.GENERIC, TransportRequest.Empty::new, requestHandler); - serviceB.registerRequestHandler(ACTION, ThreadPool.Names.GENERIC, TransportRequest.Empty::new, requestHandler); + serviceA.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, requestHandler); + serviceB.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, requestHandler); class CountingListener implements TransportMessageListener { AtomicInteger requestsReceived = new AtomicInteger(); @@ -639,7 +639,7 @@ public void testVoidMessageCompressed() throws Exception { try (MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY)) { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, (request, channel, task) -> { try { @@ -695,7 +695,7 @@ public void testHelloWorldCompressed() throws Exception { try (MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY)) { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -765,8 +765,18 @@ public void testIndexingDataCompression() throws Exception { fail(e.getMessage()); } }; - serviceA.registerRequestHandler("internal:sayHello", ThreadPool.Names.GENERIC, StringMessageRequest::new, handler); - serviceC.registerRequestHandler("internal:sayHello", ThreadPool.Names.GENERIC, StringMessageRequest::new, handler); + serviceA.registerRequestHandler( + "internal:sayHello", + threadPool.executor(ThreadPool.Names.GENERIC), + StringMessageRequest::new, + handler + ); + serviceC.registerRequestHandler( + "internal:sayHello", + threadPool.executor(ThreadPool.Names.GENERIC), + StringMessageRequest::new, + handler + ); Settings settingsWithCompress = Settings.builder() .put(TransportSettings.TRANSPORT_COMPRESS.getKey(), Compression.Enabled.INDEXING_DATA) @@ -830,7 +840,7 @@ public void handleException(TransportException exp) { public void testErrorMessage() throws InterruptedException { serviceA.registerRequestHandler( "internal:sayHelloException", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -893,7 +903,7 @@ public void testConcurrentSendRespondAndDisconnect() throws BrokenBarrierExcepti Set responseErrors = ConcurrentCollections.newConcurrentSet(); serviceA.registerRequestHandler( "internal:test", - randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC, + threadPool.executor(randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC), TestRequest::new, (request, channel, task) -> { try { @@ -912,7 +922,7 @@ public void testConcurrentSendRespondAndDisconnect() throws BrokenBarrierExcepti logger.trace("caught exception while responding from node B", e); } }; - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, TestRequest::new, ignoringRequestHandler); + serviceB.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, TestRequest::new, ignoringRequestHandler); int halfSenders = scaledRandomIntBetween(3, 10); final CyclicBarrier go = new CyclicBarrier(halfSenders * 2 + 1); @@ -1003,7 +1013,12 @@ public void onAfter() { // simulate restart of nodeB serviceB.close(); MockTransportService newService = buildService("TS_B_" + i, version1, transportVersion1, Settings.EMPTY); - newService.registerRequestHandler("internal:test", ThreadPool.Names.SAME, TestRequest::new, ignoringRequestHandler); + newService.registerRequestHandler( + "internal:test", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + ignoringRequestHandler + ); serviceB = newService; nodeB = newService.getLocalDiscoNode(); connectToNode(serviceB, nodeA); @@ -1027,7 +1042,7 @@ public void testNotifyOnShutdown() throws Exception { try { serviceA.registerRequestHandler( "internal:foobar", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { try { @@ -1060,7 +1075,7 @@ public void testNotifyOnShutdown() throws Exception { public void testTimeoutSendExceptionWithNeverSendingBackResponse() throws Exception { serviceA.registerRequestHandler( "internal:sayHelloTimeoutNoResponse", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> assertThat("moshe", equalTo(request.message)) ); // don't send back a response @@ -1105,7 +1120,7 @@ public void testTimeoutSendExceptionWithDelayedResponse() throws Exception { Semaphore inFlight = new Semaphore(Integer.MAX_VALUE); serviceA.registerRequestHandler( "internal:sayHelloTimeoutDelayedResponse", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { String message = request.message; @@ -1250,12 +1265,22 @@ public void handleResponse(StringMessageResponse response) {} public void handleException(TransportException exp) {} }; - serviceA.registerRequestHandler("internal:test", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceA.registerRequestHandler("internal:testNotSeen", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceA.registerRequestHandler("internal:testError", ThreadPool.Names.SAME, StringMessageRequest::new, handlerWithError); - serviceB.registerRequestHandler("internal:test", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceB.registerRequestHandler("internal:testNotSeen", ThreadPool.Names.SAME, StringMessageRequest::new, handler); - serviceB.registerRequestHandler("internal:testError", ThreadPool.Names.SAME, StringMessageRequest::new, handlerWithError); + serviceA.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceA.registerRequestHandler("internal:testNotSeen", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceA.registerRequestHandler( + "internal:testError", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + StringMessageRequest::new, + handlerWithError + ); + serviceB.registerRequestHandler("internal:test", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceB.registerRequestHandler("internal:testNotSeen", EsExecutors.DIRECT_EXECUTOR_SERVICE, StringMessageRequest::new, handler); + serviceB.registerRequestHandler( + "internal:testError", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + StringMessageRequest::new, + handlerWithError + ); String includeSettings; String excludeSettings; @@ -1570,13 +1595,18 @@ public void writeTo(StreamOutput out) throws IOException { } public void testVersionFrom0to1() throws Exception { - serviceB.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version1Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - assertThat(request.value2, equalTo(0)); // not set, coming from service A - Version1Response response = new Version1Response(1, 2); - channel.sendResponse(response); - assertEquals(transportVersion0, channel.getVersion()); - }); + serviceB.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version1Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + assertThat(request.value2, equalTo(0)); // not set, coming from service A + Version1Response response = new Version1Response(1, 2); + channel.sendResponse(response); + assertEquals(transportVersion0, channel.getVersion()); + } + ); Version0Request version0Request = new Version0Request(); version0Request.value1 = 1; @@ -1613,12 +1643,17 @@ public void handleException(TransportException exp) { } public void testVersionFrom1to0() throws Exception { - serviceA.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version0Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - Version0Response response = new Version0Response(1); - channel.sendResponse(response); - assertEquals(transportVersion0, channel.getVersion()); - }); + serviceA.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version0Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + Version0Response response = new Version0Response(1); + channel.sendResponse(response); + assertEquals(transportVersion0, channel.getVersion()); + } + ); Version1Request version1Request = new Version1Request(); version1Request.value1 = 1; @@ -1658,15 +1693,20 @@ public void handleException(TransportException exp) { } public void testVersionFrom1to1() throws Exception { - serviceB.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version1Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - assertThat(request.value2, equalTo(2)); - Version1Response response = new Version1Response(1, 2); - channel.sendResponse(response); - // channel versions don't make sense on DirectResponseChannel - assertThat(channel, instanceOf(TaskTransportChannel.class)); - assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); - }); + serviceB.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version1Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + assertThat(request.value2, equalTo(2)); + Version1Response response = new Version1Response(1, 2); + channel.sendResponse(response); + // channel versions don't make sense on DirectResponseChannel + assertThat(channel, instanceOf(TaskTransportChannel.class)); + assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); + } + ); Version1Request version1Request = new Version1Request(); version1Request.value1 = 1; @@ -1706,14 +1746,19 @@ public void handleException(TransportException exp) { } public void testVersionFrom0to0() throws Exception { - serviceA.registerRequestHandler("internal:version", ThreadPool.Names.SAME, Version0Request::new, (request, channel, task) -> { - assertThat(request.value1, equalTo(1)); - Version0Response response = new Version0Response(1); - channel.sendResponse(response); - // channel versions don't make sense on DirectResponseChannel - assertThat(channel, instanceOf(TaskTransportChannel.class)); - assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); - }); + serviceA.registerRequestHandler( + "internal:version", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + Version0Request::new, + (request, channel, task) -> { + assertThat(request.value1, equalTo(1)); + Version0Response response = new Version0Response(1); + channel.sendResponse(response); + // channel versions don't make sense on DirectResponseChannel + assertThat(channel, instanceOf(TaskTransportChannel.class)); + assertThat(((TaskTransportChannel) channel).getChannel(), instanceOf(TransportService.DirectResponseChannel.class)); + } + ); Version0Request version0Request = new Version0Request(); version0Request.value1 = 1; @@ -1752,7 +1797,7 @@ public void handleException(TransportException exp) { public void testMockFailToSendNoConnectRule() throws Exception { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -1808,7 +1853,7 @@ public void handleException(TransportException exp) { public void testMockUnresponsiveRule() throws InterruptedException { serviceA.registerRequestHandler( "internal:sayHello", - ThreadPool.Names.GENERIC, + threadPool.executor(ThreadPool.Names.GENERIC), StringMessageRequest::new, (request, channel, task) -> { assertThat("moshe", equalTo(request.message)); @@ -1860,11 +1905,16 @@ public void testHostOnMessages() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(2); final AtomicReference addressA = new AtomicReference<>(); final AtomicReference addressB = new AtomicReference<>(); - serviceB.registerRequestHandler("internal:action1", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - addressA.set(request.remoteAddress()); - channel.sendResponse(new TestResponse((String) null)); - latch.countDown(); - }); + serviceB.registerRequestHandler( + "internal:action1", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + addressA.set(request.remoteAddress()); + channel.sendResponse(new TestResponse((String) null)); + latch.countDown(); + } + ); serviceA.sendRequest(nodeB, "internal:action1", new TestRequest(), new TransportResponseHandler() { @Override public TestResponse read(StreamInput in) throws IOException { @@ -1902,10 +1952,15 @@ public void handleException(TransportException exp) { public void testRejectEarlyIncomingRequests() throws Exception { try (TransportService service = buildService("TS_TEST", version0, transportVersion0, null, Settings.EMPTY, false, false)) { AtomicBoolean requestProcessed = new AtomicBoolean(false); - service.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - requestProcessed.set(true); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - }); + service.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + requestProcessed.set(true); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); DiscoveryNode node = service.getLocalNode(); serviceA.close(); @@ -2145,19 +2200,19 @@ public Executor executor(ThreadPool threadPool) { } serviceB.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, new TestRequestHandler(serviceB) ); serviceC.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, new TestRequestHandler(serviceC) ); serviceA.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, new TestRequestHandler(serviceA) ); @@ -2231,7 +2286,7 @@ public Executor executor(ThreadPool threadPool) { public void testRegisterHandlerTwice() { serviceB.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2241,7 +2296,7 @@ public void testRegisterHandlerTwice() { IllegalArgumentException.class, () -> serviceB.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2251,7 +2306,7 @@ public void testRegisterHandlerTwice() { serviceA.registerRequestHandler( "internal:action1", - randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC), + threadPool.executor(randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC)), TestRequest::new, (request, message, task) -> { throw new AssertionError("boom"); @@ -2449,16 +2504,21 @@ public void run() { public void testResponseHeadersArePreserved() throws InterruptedException { List executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet()); CollectionUtil.timSort(executors); // makes sure it's reproducible - serviceA.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { + serviceA.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { - threadPool.getThreadContext().putTransient("boom", new Object()); - threadPool.getThreadContext().addResponseHeader("foo.bar", "baz"); - if ("fail".equals(request.info)) { - throw new RuntimeException("boom"); - } else { - channel.sendResponse(TransportResponse.Empty.INSTANCE); + threadPool.getThreadContext().putTransient("boom", new Object()); + threadPool.getThreadContext().addResponseHeader("foo.bar", "baz"); + if ("fail".equals(request.info)) { + throw new RuntimeException("boom"); + } else { + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } } - }); + ); CountDownLatch latch = new CountDownLatch(2); @@ -2512,9 +2572,14 @@ public void testHandlerIsInvokedOnConnectionClose() throws IOException, Interrup List executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet()); CollectionUtil.timSort(executors); // makes sure it's reproducible TransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); - serviceC.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // do nothing - }); + serviceC.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // do nothing + } + ); CountDownLatch latch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler() { @Override @@ -2576,26 +2641,31 @@ public void testConcurrentDisconnectOnNonPublishedConnection() throws IOExceptio MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); CountDownLatch receivedLatch = new CountDownLatch(1); CountDownLatch sendResponseLatch = new CountDownLatch(1); - serviceC.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); + serviceC.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); + } } - } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - }); - }); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + }); + } + ); CountDownLatch responseLatch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @Override @@ -2643,26 +2713,31 @@ public void testTransportStats() throws Exception { MockTransportService serviceC = buildService("TS_C", version0, transportVersion0, Settings.EMPTY); CountDownLatch receivedLatch = new CountDownLatch(1); CountDownLatch sendResponseLatch = new CountDownLatch(1); - serviceB.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); + serviceB.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); + } } - } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - }); - }); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + }); + } + ); CountDownLatch responseLatch = new CountDownLatch(1); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @Override @@ -2755,26 +2830,31 @@ public void testTransportStatsWithException() throws Exception { CountDownLatch sendResponseLatch = new CountDownLatch(1); Exception ex = new RuntimeException("boom"); ex.setStackTrace(new StackTraceElement[0]); - serviceB.registerRequestHandler("internal:action", ThreadPool.Names.SAME, TestRequest::new, (request, channel, task) -> { - // don't block on a network thread here - threadPool.generic().execute(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - try { - channel.sendResponse(e); - } catch (IOException e1) { - throw new UncheckedIOException(e1); + serviceB.registerRequestHandler( + "internal:action", + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TestRequest::new, + (request, channel, task) -> { + // don't block on a network thread here + threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + try { + channel.sendResponse(e); + } catch (IOException e1) { + throw new UncheckedIOException(e1); + } } - } - @Override - protected void doRun() throws Exception { - receivedLatch.countDown(); - sendResponseLatch.await(); - onFailure(ex); - } - }); - }); + @Override + protected void doRun() throws Exception { + receivedLatch.countDown(); + sendResponseLatch.await(); + onFailure(ex); + } + }); + } + ); CountDownLatch responseLatch = new CountDownLatch(1); AtomicReference receivedException = new AtomicReference<>(null); TransportResponseHandler transportResponseHandler = new TransportResponseHandler.Empty() { @@ -3120,31 +3200,41 @@ public void testFailToSendIllegalStateException() throws InterruptedException { public void testChannelToString() { final String ACTION = "internal:action"; - serviceA.registerRequestHandler(ACTION, ThreadPool.Names.SAME, TransportRequest.Empty::new, (request, channel, task) -> { - assertThat( - channel.toString(), - allOf( - containsString("DirectResponseChannel"), - containsString('{' + ACTION + '}'), - containsString("TaskTransportChannel{task=" + task.getId() + '}') - ) - ); - assertThat(new ChannelActionListener<>(channel).toString(), containsString(channel.toString())); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - }); - serviceB.registerRequestHandler(ACTION, ThreadPool.Names.SAME, TransportRequest.Empty::new, (request, channel, task) -> { - assertThat( - channel.toString(), - allOf( - containsString("TcpTransportChannel"), - containsString('{' + ACTION + '}'), - containsString("TaskTransportChannel{task=" + task.getId() + '}'), - containsString("localAddress="), - containsString(serviceB.getLocalNode().getAddress().toString()) - ) - ); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - }); + serviceA.registerRequestHandler( + ACTION, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TransportRequest.Empty::new, + (request, channel, task) -> { + assertThat( + channel.toString(), + allOf( + containsString("DirectResponseChannel"), + containsString('{' + ACTION + '}'), + containsString("TaskTransportChannel{task=" + task.getId() + '}') + ) + ); + assertThat(new ChannelActionListener<>(channel).toString(), containsString(channel.toString())); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); + serviceB.registerRequestHandler( + ACTION, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + TransportRequest.Empty::new, + (request, channel, task) -> { + assertThat( + channel.toString(), + allOf( + containsString("TcpTransportChannel"), + containsString('{' + ACTION + '}'), + containsString("TaskTransportChannel{task=" + task.getId() + '}'), + containsString("localAddress="), + containsString(serviceB.getLocalNode().getAddress().toString()) + ) + ); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); PlainActionFuture.get( f -> submitRequest( @@ -3233,7 +3323,7 @@ public void writeTo(StreamOutput out) throws IOException { final var requestSize = between(0, ByteSizeUnit.MB.toIntBytes(1)); final var responseSize = between(0, ByteSizeUnit.MB.toIntBytes(1)); - serviceB.registerRequestHandler(ACTION, ThreadPool.Names.SAME, Request::new, (request, channel, task) -> { + serviceB.registerRequestHandler(ACTION, EsExecutors.DIRECT_EXECUTOR_SERVICE, Request::new, (request, channel, task) -> { assertEquals(requestSize, request.refSize); channel.sendResponse(new Response(responseSize)); }); diff --git a/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java index 2925254ea992b..6d5542db5e529 100644 --- a/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java +++ b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java @@ -322,7 +322,12 @@ public void handleException(TransportException exp) { } private void registerRequestHandler(TransportService transportService, TransportRequestHandler handler) { - transportService.registerRequestHandler(TEST_ACTION, ThreadPool.Names.GENERIC, TestRequest::new, handler); + transportService.registerRequestHandler( + TEST_ACTION, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC), + TestRequest::new, + handler + ); } private void send( diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java index ee7924428dd0b..eb7cc9c51c62e 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -59,13 +60,15 @@ public void init() { } public void testAvailable() throws Exception { - AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); Client client = mockClient(); AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, client @@ -84,14 +87,16 @@ public void testAvailable() throws Exception { } public void testEnabled() throws Exception { - AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + AnalyticsInfoTransportAction featureSet = new AnalyticsInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); assertTrue(featureSet.enabled()); Client client = mockClient(); AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, client diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java index 4f9d803c50c25..6b0b53aedd12d 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; @@ -36,6 +37,7 @@ import static java.util.stream.Collectors.toList; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -44,6 +46,10 @@ public TransportAnalyticsStatsAction action(AnalyticsUsage usage) { TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + ClusterService clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); when(clusterService.localNode()).thenReturn(discoveryNode); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java index 93ac8bdc85896..91dc4f7d9d479 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportDeleteAutoscalingPolicyActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; @@ -39,10 +40,12 @@ public class TransportDeleteAutoscalingPolicyActionTests extends AutoscalingTestCase { public void testWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportDeleteAutoscalingPolicyAction action = new TransportDeleteAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); @@ -61,10 +64,12 @@ public void testWriteBlock() { } public void testNoWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportDeleteAutoscalingPolicyAction action = new TransportDeleteAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java index 17f97e2ddc050..12d9fcee8bd10 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingPolicyActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingLicenseChecker; @@ -37,10 +38,12 @@ public class TransportGetAutoscalingPolicyActionTests extends AutoscalingTestCase { public void testReadBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportGetAutoscalingPolicyAction action = new TransportGetAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), new AutoscalingLicenseChecker(() -> true) @@ -64,10 +67,12 @@ public void testReadBlock() { } public void testNoReadBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportGetAutoscalingPolicyAction action = new TransportGetAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), new AutoscalingLicenseChecker(() -> true) diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java index a39b3113a96ca..bca2fc4ab96af 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/TransportPutAutoscalingPolicyActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingLicenseChecker; @@ -41,10 +42,12 @@ public class TransportPutAutoscalingPolicyActionTests extends AutoscalingTestCas private static final PolicyValidator NO_VALIDATION = policy -> {}; public void testWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportPutAutoscalingPolicyAction action = new TransportPutAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_VALIDATION, @@ -65,10 +68,12 @@ public void testWriteBlock() { } public void testNoWriteBlock() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); final TransportPutAutoscalingPolicyAction action = new TransportPutAutoscalingPolicyAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), NO_VALIDATION, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java index 7a7d375781e16..1d81549f7adec 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; @@ -48,8 +49,9 @@ public void init() { } public void testAvailable() { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -64,8 +66,9 @@ public void testAvailable() { public void testEnabled() { Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState @@ -73,13 +76,14 @@ public void testEnabled() { assertThat(featureSet.enabled(), equalTo(false)); settings = Settings.builder().put("xpack.ccr.enabled", true); - featureSet = new CCRInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState); + featureSet = new CCRInfoTransportAction(transportService, mock(ActionFilters.class), settings.build(), licenseState); assertThat(featureSet.enabled(), equalTo(true)); } public void testName() { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); CCRInfoTransportAction featureSet = new CCRInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -137,10 +141,12 @@ public void testUsageStats() throws Exception { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build(); Mockito.when(clusterService.state()).thenReturn(clusterState); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new CCRUsageTransportAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, Settings.EMPTY, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java index 95ab1c2323f7d..f2423ca4d1d16 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; @@ -57,6 +58,7 @@ public void testActionNames() { final TransportService transportService = mock(TransportService.class); final CcrRestoreSourceService ccrRestoreSourceService = mock(CcrRestoreSourceService.class); + MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new ClearCcrRestoreSessionAction.TransportAction(actionFilters, transportService, ccrRestoreSourceService); assertThat(action.actionName, equalTo(ClearCcrRestoreSessionAction.NAME)); @@ -87,6 +89,7 @@ public void testRequestedShardIdMustBeConsistentWithSessionShardId() { } }).when(ccrRestoreSourceService).ensureSessionShardIdConsistency(anyString(), any()); + MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new ClearCcrRestoreSessionAction.TransportAction(actionFilters, transportService, ccrRestoreSourceService); final String sessionUUID = UUIDs.randomBase64UUID(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java index 7357873bba0f2..307bb6e67b10f 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; @@ -62,6 +63,7 @@ public void testActionNames() { final TransportService transportService = mock(TransportService.class); final CcrRestoreSourceService ccrRestoreSourceService = mock(CcrRestoreSourceService.class); + MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new GetCcrRestoreFileChunkAction.TransportAction( bigArrays, transportService, @@ -103,6 +105,7 @@ public void testRequestedShardIdMustBeConsistentWithSessionShardId() { } }).when(ccrRestoreSourceService).ensureSessionShardIdConsistency(anyString(), any()); + MockUtils.setupTransportServiceWithThreadpoolExecutor(transportService); final var action = new GetCcrRestoreFileChunkAction.TransportAction( bigArrays, transportService, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java index 7eca64b82cbb4..1c09fcb746f49 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java @@ -135,7 +135,7 @@ public TransportTermsEnumAction( transportService.registerRequestHandler( transportShardAction, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, NodeTermsEnumRequest::new, new NodeTransportHandler() ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java index 0801ae1834954..b457a77e38f01 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import java.util.ArrayList; @@ -59,8 +60,9 @@ public void testDoExecute() throws Exception { }); } + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); TransportXPackInfoAction action = new TransportXPackInfoAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), licenseService, client diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java index e60f8c1ccae6e..190ac8d388bcf 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ESTestCase; @@ -33,6 +34,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; public class TransportNodeDeprecationCheckActionTests extends ESTestCase { @@ -62,6 +64,11 @@ public void testNodeOperation() { DiscoveryNode node = Mockito.mock(DiscoveryNode.class); when(node.getId()).thenReturn("mock-node"); TransportService transportService = Mockito.mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + when(transportService.getLocalNode()).thenReturn(node); PluginsService pluginsService = Mockito.mock(PluginsService.class); ActionFilters actionFilters = Mockito.mock(ActionFilters.class); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java index 89ff330051418..76577b12cd8d9 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -28,7 +29,8 @@ public class InternalExecutePolicyActionTests extends ESTestCase { @Before public void instantiateTransportAction() { - transportAction = new InternalExecutePolicyAction.Transport(mock(TransportService.class), mock(ActionFilters.class), null, null); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + transportAction = new InternalExecutePolicyAction.Transport(transportService, mock(ActionFilters.class), null, null); } public void testSelectNodeForPolicyExecution() { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java index c350465f0f725..c9d2f97eb5624 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -54,12 +55,14 @@ public void init() throws Exception { } public void testAvailable() { - EqlInfoTransportAction featureSet = new EqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + EqlInfoTransportAction featureSet = new EqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - EqlInfoTransportAction featureSet = new EqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + EqlInfoTransportAction featureSet = new EqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } @@ -93,10 +96,12 @@ public void testUsageStats() throws Exception { when(mockNode.getId()).thenReturn("mocknode"); when(clusterService.localNode()).thenReturn(mockNode); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new EqlUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, client diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java index 8a39bc5110c93..53d5a66de7b66 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverTaskRunner.java @@ -36,9 +36,9 @@ public class DriverTaskRunner { public static final String ACTION_NAME = "internal:data/read/esql/compute"; private final TransportService transportService; - public DriverTaskRunner(TransportService transportService, String executorName) { + public DriverTaskRunner(TransportService transportService, Executor executor) { this.transportService = transportService; - transportService.registerRequestHandler(ACTION_NAME, executorName, DriverRequest::new, new DriverRequestHandler()); + transportService.registerRequestHandler(ACTION_NAME, executor, DriverRequest::new, new DriverRequestHandler()); } public void executeDrivers(Task parentTask, List drivers, Executor executor, ActionListener listener) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java index c3efd67579d66..6db4a8c4fe37d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java @@ -79,13 +79,13 @@ public ExchangeService(Settings settings, ThreadPool threadPool, String executor public void registerTransportHandler(TransportService transportService) { transportService.registerRequestHandler( EXCHANGE_ACTION_NAME, - requestExecutorName, + threadPool.executor(requestExecutorName), ExchangeRequest::new, new ExchangeTransportAction() ); transportService.registerRequestHandler( OPEN_EXCHANGE_ACTION_NAME, - requestExecutorName, + threadPool.executor(requestExecutorName), OpenExchangeRequest::new, new OpenExchangeRequestHandler() ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 8aff85d088d6f..4d783b9a1012c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -102,12 +102,7 @@ public EnrichLookupService(ClusterService clusterService, SearchService searchSe this.searchService = searchService; this.transportService = transportService; this.executor = transportService.getThreadPool().executor(EsqlPlugin.ESQL_THREAD_POOL_NAME); - transportService.registerRequestHandler( - LOOKUP_ACTION_NAME, - EsqlPlugin.ESQL_THREAD_POOL_NAME, - LookupRequest::new, - new TransportHandler() - ); + transportService.registerRequestHandler(LOOKUP_ACTION_NAME, this.executor, LookupRequest::new, new TransportHandler()); } public void lookupAsync( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index f725e4fe60175..27634bf0d4eaa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -103,14 +103,9 @@ public ComputeService( this.searchService = searchService; this.transportService = transportService; this.bigArrays = bigArrays.withCircuitBreaking(); - transportService.registerRequestHandler( - DATA_ACTION_NAME, - ESQL_THREAD_POOL_NAME, - DataNodeRequest::new, - new DataNodeRequestHandler() - ); this.esqlExecutor = threadPool.executor(ESQL_THREAD_POOL_NAME); - this.driverRunner = new DriverTaskRunner(transportService, ESQL_THREAD_POOL_NAME); + transportService.registerRequestHandler(DATA_ACTION_NAME, this.esqlExecutor, DataNodeRequest::new, new DataNodeRequestHandler()); + this.driverRunner = new DriverTaskRunner(transportService, this.esqlExecutor); this.exchangeService = exchangeService; this.enrichLookupService = enrichLookupService; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java index 7f0bdbab6add3..f0b3a89a4444b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java @@ -43,11 +43,17 @@ public class EsqlInfoTransportActionTests extends ESTestCase { private ThreadPool threadPool; + private TransportService transportService; private Client client; @Before public void init() { threadPool = new TestThreadPool(getTestName()); + + // TODO: temporary, remove in #97879 + transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); } @@ -58,12 +64,12 @@ public void shutdown() { } public void testAvailable() { - EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + EsqlInfoTransportAction featureSet = new EsqlInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } @@ -98,7 +104,7 @@ public void testUsageStats() throws Exception { when(clusterService.localNode()).thenReturn(mockNode); var usageAction = new EsqlUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, threadPool, mock(ActionFilters.class), diff --git a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java b/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java index f1bad49abda68..90034c9620923 100644 --- a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java +++ b/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -33,8 +34,9 @@ public void init() throws Exception { } public void testAvailable() throws Exception { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GraphInfoTransportAction featureSet = new GraphInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -44,7 +46,7 @@ public void testAvailable() throws Exception { assertThat(featureSet.available(), is(available)); var usageAction = new GraphUsageTransportAction( - mock(TransportService.class), + transportService, null, mock(ThreadPool.class), mock(ActionFilters.class), @@ -73,8 +75,10 @@ public void testEnabled() throws Exception { } else { settings.put("xpack.graph.enabled", enabled); } + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); GraphInfoTransportAction featureSet = new GraphInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState @@ -82,7 +86,7 @@ public void testEnabled() throws Exception { assertThat(featureSet.enabled(), is(enabled)); GraphUsageTransportAction usageAction = new GraphUsageTransportAction( - mock(TransportService.class), + transportService, null, mock(ThreadPool.class), mock(ActionFilters.class), diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java index ae031a050d3d5..0ea85bab36a4e 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportPutSamlServiceProviderActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.idp.saml.idp.SamlIdentityProvider; import org.elasticsearch.xpack.idp.saml.sp.SamlServiceProviderDocument; @@ -57,10 +58,10 @@ public void setupMocks() { index = mock(SamlServiceProviderIndex.class); idp = mock(SamlIdentityProvider.class); when(idp.getAllowedNameIdFormats()).thenReturn(Set.of(TRANSIENT)); - now = Instant.ofEpochMilli(System.currentTimeMillis() + randomLongBetween(-500_000, 500_000)); final Clock clock = Clock.fixed(now, randomZone()); - action = new TransportPutSamlServiceProviderAction(mock(TransportService.class), mock(ActionFilters.class), index, idp, clock); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + action = new TransportPutSamlServiceProviderAction(transportService, mock(ActionFilters.class), index, idp, clock); } public void testRegisterNewServiceProvider() throws Exception { diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java index 107421ce12a31..d81faf6a398d7 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleInfoTransportActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; @@ -52,18 +53,14 @@ public void init() throws Exception { } public void testAvailable() { - IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), equalTo(true)); } public void testName() { - IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.name(), equalTo("ilm")); } @@ -94,13 +91,9 @@ public void testUsageStats() throws Exception { ClusterState clusterState = buildClusterState(policies, indexPolicies); Mockito.when(clusterService.state()).thenReturn(clusterState); - var usageAction = new IndexLifecycleUsageTransportAction( - mock(TransportService.class), - null, - mock(ThreadPool.class), - mock(ActionFilters.class), - null - ); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + var usageAction = new IndexLifecycleUsageTransportAction(transportService, null, threadPool, mock(ActionFilters.class), null); PlainActionFuture future = new PlainActionFuture<>(); usageAction.masterOperation(null, null, clusterState, future); IndexLifecycleFeatureSetUsage ilmUsage = (IndexLifecycleFeatureSetUsage) future.get().getUsage(); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java index feb66c12c2613..3e0f19246dfe3 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction; @@ -20,10 +21,12 @@ public class TransportDeleteLifecycleActionTests extends ESTestCase { public void testReservedStateHandler() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportDeleteLifecycleAction putAction = new TransportDeleteLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java index ef1d65f2e4950..8467e46630876 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -46,10 +47,12 @@ public void testIsNoop() { } public void testReservedStateHandler() throws Exception { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportPutLifecycleAction putAction = new TransportPutLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), mock(NamedXContentRegistry.class), diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java index cdc50f28fc306..2c19a79ac5af0 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.StopILMRequest; @@ -34,10 +35,12 @@ public class TransportStopILMActionTests extends ESTestCase { public void testStopILMClusterStatePriorityIsImmediate() { ClusterService clusterService = mock(ClusterService.class); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportStopILMAction transportStopILMAction = new TransportStopILMAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java index d72d079d4ae19..56b7965be3686 100644 --- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java +++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportDeletePipelineActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportService; @@ -27,8 +28,9 @@ public class TransportDeletePipelineActionTests extends ESTestCase { public void testDeletePipelineWithMissingIndex() throws Exception { try (Client client = getFailureClient(new IndexNotFoundException("missing .logstash"))) { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportDeletePipelineAction action = new TransportDeletePipelineAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), client ); diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java index d8b4b167bcc46..d8a4d048f1fe4 100644 --- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java +++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/action/TransportGetPipelineActionTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLogAppender; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.transport.RemoteTransportException; import org.elasticsearch.transport.TransportService; @@ -100,11 +101,8 @@ public void onFailure(Exception e) { try (Client client = getMockClient(multiGetResponse)) { Loggers.addAppender(logger, mockLogAppender); - TransportGetPipelineAction action = new TransportGetPipelineAction( - mock(TransportService.class), - mock(ActionFilters.class), - client - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + TransportGetPipelineAction action = new TransportGetPipelineAction(transportService, mock(ActionFilters.class), client); action.doExecute(null, request, testActionListener); } finally { Loggers.removeAppender(logger, mockLogAppender); @@ -152,8 +150,9 @@ public void onFailure(Exception e) { } }; + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); try (Client client = getMockClient(searchResponse)) { - new TransportGetPipelineAction(mock(TransportService.class), mock(ActionFilters.class), client).doExecute( + new TransportGetPipelineAction(transportService, mock(ActionFilters.class), client).doExecute( null, request, testActionListener @@ -165,8 +164,9 @@ public void onFailure(Exception e) { public void testMissingIndexHandling() throws Exception { try (Client failureClient = getFailureClient(new IndexNotFoundException("foo"))) { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportGetPipelineAction action = new TransportGetPipelineAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), failureClient ); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java index 418d277977c16..2ff9bb2906a5c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.ingest.IngestStats; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -145,10 +146,12 @@ private MachineLearningUsageTransportAction newUsageAction( boolean isDataFrameAnalyticsEnabled, boolean isNlpEnabled ) { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); return new MachineLearningUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), TestEnvironment.newEnvironment(settings), @@ -162,8 +165,9 @@ private MachineLearningUsageTransportAction newUsageAction( } public void testAvailable() throws Exception { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); MachineLearningInfoTransportAction featureSet = new MachineLearningInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), commonSettings, licenseState @@ -191,9 +195,11 @@ public void testEnabled() throws Exception { enabled = randomBoolean(); settings.put("xpack.ml.enabled", enabled); } + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); boolean expected = enabled; MachineLearningInfoTransportAction featureSet = new MachineLearningInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java index c55b78c07939a..67da449af850f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportCloseJobActionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.MlTasks; @@ -337,10 +338,12 @@ public static void addTask( } private TransportCloseJobAction createAction() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); return new TransportCloseJobAction( - mock(TransportService.class), + transportService, client, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), clusterService, mock(AnomalyDetectionAuditor.class), diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java index f85a13fcb3935..26409b1ada849 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java @@ -38,6 +38,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; public class TransportDeleteExpiredDataActionTests extends ESTestCase { @@ -59,6 +60,10 @@ public void remove(float requestsPerSec, ActionListener listener, Boole public void setup() { threadPool = new TestThreadPool("TransportDeleteExpiredDataActionTests thread pool"); TransportService transportService = mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + Client client = mock(Client.class); ClusterService clusterService = mock(ClusterService.class); auditor = mock(AnomalyDetectionAuditor.class); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java index 1c0243fcab098..c8096a7b1f384 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java @@ -81,8 +81,12 @@ public void testOperation() { } private TransportFinalizeJobExecutionAction createAction(ClusterService clusterService) { + // TODO: temporary, remove in #97879 + TransportService transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + return new TransportFinalizeJobExecutionAction( - mock(TransportService.class), + transportService, clusterService, threadPool, mock(ActionFilters.class), diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java index 79a376b90040f..4d242db394d10 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; @@ -44,18 +45,14 @@ public class MonitoringInfoTransportActionTests extends ESTestCase { private final Exporters exporters = mock(Exporters.class); public void testAvailable() { - MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testMonitoringEnabledByDefault() { - MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class) - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } @@ -94,10 +91,12 @@ public void testUsage() throws Exception { when(exporters.getEnabledExporters()).thenReturn(exporterList); when(monitoring.isMonitoringActive()).thenReturn(collectionEnabled); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new MonitoringUsageTransportAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, new MonitoringUsageServices(monitoring, exporters) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java index 3676fc5d2e7a0..d2a694c04ae34 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -98,6 +99,10 @@ public void setUpMocks() { transportService = mock(TransportService.class); filters = mock(ActionFilters.class); + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + when(transportService.getTaskManager()).thenReturn(taskManager); when(taskManager.register(anyString(), eq(MonitoringBulkAction.NAME), any(TaskAwareRequest.class))).thenReturn(mock(Task.class)); when(filters.filters()).thenReturn(new ActionFilter[0]); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java index dd78eb95e7f3d..9fe634a178179 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -49,7 +50,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportService; @@ -105,7 +105,7 @@ public TransportRollupSearchAction( transportService.registerRequestHandler( actionName, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, true, SearchRequest::new, diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java index ab7420f0b50ee..b0881eb350d5a 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -25,23 +26,21 @@ public class RollupInfoTransportActionTests extends ESTestCase { public void testAvailable() { - RollupInfoTransportAction featureSet = new RollupInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + RollupInfoTransportAction featureSet = new RollupInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabledDefault() { - RollupInfoTransportAction featureSet = new RollupInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + RollupInfoTransportAction featureSet = new RollupInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); } public void testUsage() throws ExecutionException, InterruptedException, IOException { - var usageAction = new RollupUsageTransportAction( - mock(TransportService.class), - null, - mock(ThreadPool.class), - mock(ActionFilters.class), - null - ); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + var usageAction = new RollupUsageTransportAction(transportService, null, threadPool, mock(ActionFilters.class), null); PlainActionFuture future = new PlainActionFuture<>(); usageAction.masterOperation(null, null, null, future); XPackFeatureSet.Usage rollupUsage = future.get().getUsage(); diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java index ddba0395382f3..985bec3ba8c77 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -1108,7 +1109,7 @@ private static MockTransportService startTransport( try { service.registerRequestHandler( ClusterStateAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, ClusterStateRequest::new, (request, channel, task) -> { capturedHeaders.add( @@ -1121,7 +1122,7 @@ private static MockTransportService startTransport( ); service.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> { capturedHeaders.add( @@ -1132,7 +1133,7 @@ private static MockTransportService startTransport( ); service.registerRequestHandler( SearchShardsAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, SearchShardsRequest::new, (request, channel, task) -> { capturedHeaders.add( @@ -1141,31 +1142,36 @@ private static MockTransportService startTransport( channel.sendResponse(new SearchShardsResponse(List.of(), List.of(), Collections.emptyMap())); } ); - service.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> { - capturedHeaders.add( - new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders())) - ); - channel.sendResponse( - new SearchResponse( - new InternalSearchResponse( - new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - null, - false, + service.registerRequestHandler( + SearchAction.NAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + SearchRequest::new, + (request, channel, task) -> { + capturedHeaders.add( + new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders())) + ); + channel.sendResponse( + new SearchResponse( + new InternalSearchResponse( + new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), + InternalAggregations.EMPTY, + null, + null, + false, + null, + 1 + ), null, - 1 - ), - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) - ); - }); + 1, + 1, + 0, + 100, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ) + ); + } + ); service.start(); service.acceptIncomingRequests(); success = true; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java index 4b750fb7d2ab6..88f233087e1dd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ToXContent; @@ -78,24 +79,18 @@ public void init() throws Exception { } public void testAvailable() { - SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class), - settings - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), settings); assertThat(featureSet.available(), is(true)); } public void testEnabled() { - SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class), - settings - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), settings); assertThat(featureSet.enabled(), is(true)); Settings disabled = Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), false).build(); - featureSet = new SecurityInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class), disabled); + featureSet = new SecurityInfoTransportAction(transportService, mock(ActionFilters.class), disabled); assertThat(featureSet.enabled(), is(false)); } @@ -371,10 +366,12 @@ private void configureRoleMappingStoreUsage(boolean roleMappingStoreEnabled) { } private SecurityUsageTransportAction newUsageAction(Settings settings) { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); return new SecurityUsageTransportAction( - mock(TransportService.class), + transportService, null, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, settings, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java index e2d1fc3a24568..f94acab50b6b5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportCreateCrossClusterApiKeyActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyResponse; @@ -42,12 +43,8 @@ public void setUp() throws Exception { super.setUp(); apiKeyService = mock(ApiKeyService.class); securityContext = mock(SecurityContext.class); - action = new TransportCreateCrossClusterApiKeyAction( - mock(TransportService.class), - mock(ActionFilters.class), - apiKeyService, - securityContext - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + action = new TransportCreateCrossClusterApiKeyAction(transportService, mock(ActionFilters.class), apiKeyService, securityContext); } public void testApiKeyWillBeCreatedWithEmptyUserRoleDescriptors() throws IOException { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java index 5d68dea7a146e..f9e219f168718 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java @@ -61,6 +61,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; public class TransportGrantApiKeyActionTests extends ESTestCase { @@ -69,6 +70,7 @@ public class TransportGrantApiKeyActionTests extends ESTestCase { private ApiKeyUserRoleDescriptorResolver resolver; private AuthenticationService authenticationService; private ThreadPool threadPool; + private TransportService transportService; private AuthorizationService authorizationService; @Before @@ -81,8 +83,12 @@ public void setupMocks() throws Exception { threadPool = new TestThreadPool("TP-" + getTestName()); final ThreadContext threadContext = threadPool.getThreadContext(); + // TODO: temporary, remove in #97879 + TransportService transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + action = new TransportGrantApiKeyAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), threadContext, authenticationService, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java index 711c604206823..7ce920506d7d1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportUpdateCrossClusterApiKeyActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.apikey.ApiKey; @@ -55,8 +56,9 @@ public void testExecute() throws IOException { () -> AuthenticationTestHelper.builder().build() ); when(securityContext.getAuthentication()).thenReturn(authentication); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportUpdateCrossClusterApiKeyAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), apiKeyService, securityContext @@ -120,8 +122,9 @@ public void testExecute() throws IOException { public void testAuthenticationCheck() { final SecurityContext securityContext = mock(SecurityContext.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportUpdateCrossClusterApiKeyAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), mock(ApiKeyService.class), securityContext diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java index ed01d0ca6fc40..dca8633d7c1eb 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java @@ -64,6 +64,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportProfileHasPrivilegesActionTests extends ESTestCase { @@ -80,6 +81,10 @@ public class TransportProfileHasPrivilegesActionTests extends ESTestCase { public void setup() { threadPool = new TestThreadPool(TransportProfileHasPrivilegesActionTests.class.getSimpleName()); transportService = mock(TransportService.class); + + // TODO: temporary, remove in #97879 + when(transportService.getThreadPool()).thenReturn(threadPool); + actionFilters = mock(ActionFilters.class); authorizationService = mock(AuthorizationService.class); nativePrivilegeStore = mock(NativePrivilegeStore.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java index df414207da8ee..92bef206a9d6a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest; @@ -50,10 +51,13 @@ public void setup() { fileRealm = mockRealm("file"); final Realms realms = mockRealms(List.of(nativeRealm, fileRealm)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + action = new TransportClearRealmCacheAction( - mock(ThreadPool.class), + threadPool, mockClusterService(), - mock(TransportService.class), + transportService, mock(ActionFilters.class), realms, authenticationService diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java index ac7680ea2b574..5194ff032c5ba 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportCreateServiceAccountTokenActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.service.CreateServiceAccountTokenRequest; @@ -40,8 +41,10 @@ public class TransportCreateServiceAccountTokenActionTests extends ESTestCase { public void init() throws IOException { serviceAccountService = mock(ServiceAccountService.class); securityContext = mock(SecurityContext.class); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportCreateServiceAccountTokenAction = new TransportCreateServiceAccountTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), serviceAccountService, securityContext diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java index 8b0530e48c560..2050d31638fe9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportDeleteServiceAccountTokenActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.DeleteServiceAccountTokenRequest; import org.elasticsearch.xpack.core.security.action.service.DeleteServiceAccountTokenResponse; @@ -32,8 +33,9 @@ public class TransportDeleteServiceAccountTokenActionTests extends ESTestCase { @Before public void init() { serviceAccountService = mock(ServiceAccountService.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportDeleteServiceAccountTokenAction = new TransportDeleteServiceAccountTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), serviceAccountService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java index 87ef1ea24559e..e8d178efe6b4a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountRequest; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountResponse; @@ -31,8 +32,9 @@ public class TransportGetServiceAccountActionTests extends ESTestCase { @Before public void init() { + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportGetServiceAccountAction = new TransportGetServiceAccountAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()) ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java index da6508ac79ef5..ec68ed7a2776f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountCredentialsActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountCredentialsRequest; @@ -58,8 +59,10 @@ public void init() throws UnknownHostException { when(transport.boundAddress()).thenReturn(new BoundTransportAddress(new TransportAddress[] { transportAddress }, transportAddress)); final Settings settings = builder.build(); serviceAccountService = mock(ServiceAccountService.class); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); transportGetServiceAccountCredentialsAction = new TransportGetServiceAccountCredentialsAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), serviceAccountService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java index bd63dd65824bd..dcdeace6b06b6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java @@ -88,6 +88,7 @@ public class TransportCreateTokenActionTests extends ESTestCase { .build(); private ThreadPool threadPool; + private TransportService transportService; private Client client; private SecurityIndexManager securityIndex; private ClusterService clusterService; @@ -99,6 +100,11 @@ public class TransportCreateTokenActionTests extends ESTestCase { @Before public void setupClient() { threadPool = new TestThreadPool(getTestName()); + + // TODO: temporary, remove in #97879 + transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + client = mock(Client.class); idxReqReference = new AtomicReference<>(); authenticationService = mock(AuthenticationService.class); @@ -221,7 +227,7 @@ public void testClientCredentialsCreatesWithoutRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -262,7 +268,7 @@ public void testPasswordGrantTypeCreatesWithRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -305,7 +311,7 @@ public void testKerberosGrantTypeCreatesWithRefreshToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -358,7 +364,7 @@ public void testKerberosGrantTypeWillFailOnBase64DecodeError() throws Exception final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, @@ -397,7 +403,7 @@ public void testServiceAccountCannotCreateOAuthToken() throws Exception { final TransportCreateTokenAction action = new TransportCreateTokenAction( threadPool, - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService, authenticationService, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java index a24361078ff26..f07bb7dc365ae 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java @@ -56,6 +56,7 @@ public class TransportInvalidateTokenActionTests extends ESTestCase { .build(); private ThreadPool threadPool; + private TransportService transportService; private Client client; private SecurityIndexManager securityIndex; private ClusterService clusterService; @@ -65,6 +66,11 @@ public class TransportInvalidateTokenActionTests extends ESTestCase { @Before public void setup() { threadPool = new TestThreadPool(getTestName()); + + // TODO: temporary, remove in #97879 + transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); + securityContext = new SecurityContext(Settings.EMPTY, threadPool.getThreadContext()); client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); @@ -91,7 +97,7 @@ public void testInvalidateTokensWhenIndexUnavailable() throws Exception { clusterService ); final TransportInvalidateTokenAction action = new TransportInvalidateTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService ); @@ -137,7 +143,7 @@ public void testInvalidateTokensWhenIndexClosed() throws Exception { clusterService ); final TransportInvalidateTokenAction action = new TransportInvalidateTokenAction( - mock(TransportService.class), + transportService, new ActionFilters(Collections.emptySet()), tokenService ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java index b4a79e070e927..9f7815a1c9891 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest; @@ -57,8 +58,10 @@ public void testHasPrivilegesRequestDoesNotAllowDLSRoleQueryBasedIndicesPrivileg .build(false); when(context.getAuthentication()).thenReturn(authentication); threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final TransportHasPrivilegesAction transportHasPrivilegesAction = new TransportHasPrivilegesAction( - mock(TransportService.class), + transportService, new ActionFilters(Set.of()), mock(AuthorizationService.class), mock(NativePrivilegeStore.class), @@ -110,8 +113,10 @@ public void testRequiresSameUser() { ); return null; }).when(authorizationService).checkPrivileges(any(), any(), anyCollection(), anyActionListener()); + + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); final var action = new TransportHasPrivilegesAction( - mock(TransportService.class), + transportService, new ActionFilters(Set.of()), authorizationService, privilegeStore, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java index 600d4d1ba5fa4..2a15aa09ddccd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.IOUtils; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.mocksocket.MockSocket; @@ -159,7 +160,7 @@ public TransportRequestHandler interceptHandler( DiscoveryNode remoteNode = remoteTransportService.getLocalDiscoNode(); remoteTransportService.registerRequestHandler( RemoteClusterNodesAction.NAME, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, RemoteClusterNodesAction.Request::new, (request, channel, task) -> channel.sendResponse(new RemoteClusterNodesAction.Response(List.of(remoteNode))) ); diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java index 67ce8c255fd94..cf28bf9922b24 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportDeleteShutdownNodeActionTests.java @@ -19,8 +19,8 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.shutdown.TransportDeleteShutdownNodeAction.DeleteShutdownNodeExecutor; import org.elasticsearch.xpack.shutdown.TransportDeleteShutdownNodeAction.DeleteShutdownNodeTask; import org.junit.Before; @@ -55,7 +55,7 @@ public void init() { MockitoAnnotations.openMocks(this); // TODO: it takes almost 2 seconds to create these mocks....WHY?!? var threadPool = mock(ThreadPool.class); - var transportService = mock(TransportService.class); + var transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); clusterService = mock(ClusterService.class); var actionFilters = mock(ActionFilters.class); var indexNameExpressionResolver = mock(IndexNameExpressionResolver.class); diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java index c264ad6ad1d03..cbd51ceebc729 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeActionTests.java @@ -19,8 +19,8 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.shutdown.TransportPutShutdownNodeAction.PutShutdownNodeExecutor; import org.elasticsearch.xpack.shutdown.TransportPutShutdownNodeAction.PutShutdownNodeTask; import org.junit.Before; @@ -61,7 +61,7 @@ public void init() { MockitoAnnotations.openMocks(this); // TODO: it takes almost 2 seconds to create these mocks....WHY?!? var threadPool = mock(ThreadPool.class); - var transportService = mock(TransportService.class); + var transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); clusterService = mock(ClusterService.class); var actionFilters = mock(ActionFilters.class); var indexNameExpressionResolver = mock(IndexNameExpressionResolver.class); diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java index 4c71683a972a1..954b5ba024418 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.reservedstate.service.ReservedStateUpdateTask; import org.elasticsearch.reservedstate.service.ReservedStateUpdateTaskExecutor; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentParser; @@ -363,10 +364,12 @@ public void testOperatorControllerFromJSONContent() throws IOException { } public void testDeleteSLMReservedStateHandler() { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var deleteAction = new TransportDeleteSnapshotLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); @@ -377,10 +380,12 @@ public void testDeleteSLMReservedStateHandler() { } public void testPutSLMReservedStateHandler() throws Exception { + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var putAction = new TransportPutSnapshotLifecycleAction( - mock(TransportService.class), + transportService, mock(ClusterService.class), - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java index 0294c14f57494..5d0e002c49036 100644 --- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java +++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.action.StopILMAction; @@ -33,11 +34,12 @@ public class TransportStopSLMActionTests extends ESTestCase { public void testStopILMClusterStatePriorityIsImmediate() { ClusterService clusterService = mock(ClusterService.class); - + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); TransportStopSLMAction transportStopSLMAction = new TransportStopSLMAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class) ); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java index 83eb9d49a4e21..ede64decbca33 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialInfoTransportActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; @@ -56,13 +57,15 @@ public void init() { } public void testAvailable() throws Exception { - SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); var usageAction = new SpatialUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, mockClient() @@ -80,14 +83,16 @@ public void testAvailable() throws Exception { } public void testEnabled() throws Exception { - SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + SpatialInfoTransportAction featureSet = new SpatialInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.enabled(), is(true)); assertTrue(featureSet.enabled()); SpatialUsageTransportAction usageAction = new SpatialUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, mockClient() diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java index 4f42305984b57..1b91f686a7d8d 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/action/SpatialStatsTransportActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -47,8 +48,8 @@ public class SpatialStatsTransportActionTests extends ESTestCase { @Before public void mockServices() { - transportService = mock(TransportService.class); threadPool = mock(ThreadPool.class); + transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java index f95fd337fe5b7..01bad2c3dee60 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -57,11 +58,8 @@ public void init() throws Exception { } public void testAvailable() { - SqlInfoTransportAction featureSet = new SqlInfoTransportAction( - mock(TransportService.class), - mock(ActionFilters.class), - licenseState - ); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + SqlInfoTransportAction featureSet = new SqlInfoTransportAction(transportService, mock(ActionFilters.class), licenseState); assertThat(featureSet.available(), is(true)); } @@ -95,10 +93,12 @@ public void testUsageStats() throws Exception { when(mockNode.getId()).thenReturn("mocknode"); when(clusterService.localNode()).thenReturn(mockNode); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); var usageAction = new SqlUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, licenseState, diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java index 5c6f875415384..d2c23f768cef6 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; @@ -29,12 +30,14 @@ public class TransformInfoTransportActionTests extends ESTestCase { public void testAvailable() { - TransformInfoTransportAction featureSet = new TransformInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + TransformInfoTransportAction featureSet = new TransformInfoTransportAction(transportService, mock(ActionFilters.class)); assertThat(featureSet.available(), is(true)); } public void testEnabledDefault() { - TransformInfoTransportAction featureSet = new TransformInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class)); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(); + TransformInfoTransportAction featureSet = new TransformInfoTransportAction(transportService, mock(ActionFilters.class)); assertTrue(featureSet.enabled()); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java index 0b796ee3d3f7d..ac683dca5bf26 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; @@ -55,20 +56,23 @@ public class WatcherInfoTransportActionTests extends ESTestCase { private MockLicenseState licenseState; private Client client; + private ThreadPool threadPool; + private TransportService transportService; @Before public void init() throws Exception { licenseState = mock(MockLicenseState.class); client = mock(Client.class); - ThreadPool threadPool = mock(ThreadPool.class); + threadPool = mock(ThreadPool.class); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); when(threadPool.getThreadContext()).thenReturn(threadContext); when(client.threadPool()).thenReturn(threadPool); + transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); } public void testAvailable() { WatcherInfoTransportAction featureSet = new WatcherInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), Settings.EMPTY, licenseState @@ -89,7 +93,7 @@ public void testEnabled() { settings.put("xpack.watcher.enabled", enabled); } WatcherInfoTransportAction featureSet = new WatcherInfoTransportAction( - mock(TransportService.class), + transportService, mock(ActionFilters.class), settings.build(), licenseState @@ -130,9 +134,9 @@ public void testUsageStats() throws Exception { when(clusterService.localNode()).thenReturn(mockNode); var usageAction = new WatcherUsageTransportAction( - mock(TransportService.class), + transportService, clusterService, - mock(ThreadPool.class), + threadPool, mock(ActionFilters.class), null, Settings.EMPTY, diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java index 79c93cd5b913a..338f4e56f5663 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportAckWatchActionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.license.TestUtils; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.watcher.WatcherMetadata; @@ -56,8 +57,8 @@ public class TransportAckWatchActionTests extends ESTestCase { @Before public void setupAction() { - TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); when(threadPool.getThreadContext()).thenReturn(threadContext); WatchParser watchParser = mock(WatchParser.class); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java index 09cf9daaf3685..95272619fac5d 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportPutWatchActionTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.license.TestUtils; import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ClientHelper; @@ -63,7 +64,7 @@ public void setupAction() throws Exception { ThreadPool threadPool = mock(ThreadPool.class); when(threadPool.getThreadContext()).thenReturn(threadContext); - TransportService transportService = mock(TransportService.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); WatchParser parser = mock(WatchParser.class); when(parser.parseWithSecrets(eq("_id"), eq(false), any(), any(), any(), anyBoolean(), anyLong(), anyLong())).thenReturn(watch); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java index 8d07d1eaf6043..6dfdf47a7a735 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherStatsActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -44,8 +45,8 @@ public class TransportWatcherStatsActionTests extends ESTestCase { @Before public void setupTransportAction() { - TransportService transportService = mock(TransportService.class); ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); ClusterService clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); From dbbe44300914cfb59063bce0815095f6bf94fa4d Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Fri, 8 Sep 2023 16:27:15 -0700 Subject: [PATCH 55/69] Add sort guarantee to CSV query --- .../plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec index f9d7a846dd99a..ab764084bd309 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec @@ -159,6 +159,7 @@ from employees | keep emp_no | eval sum = emp_no + 1 | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | limit 3 +| sort emp_no ; emp_no:i | sum:i From 7becfbf60f36808d7cbc62de695c9706b3e7fb7f Mon Sep 17 00:00:00 2001 From: David Turner Date: Sun, 10 Sep 2023 21:45:32 +0100 Subject: [PATCH 56/69] Mute test for #99394 (#99395) --- .../esql/qa/testFixtures/src/main/resources/eval.csv-spec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec index ab764084bd309..11111db4620d9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec @@ -110,7 +110,8 @@ true | false | false | true ; -repetitiveEval +// AwaitsFix https://github.com/elastic/elasticsearch/issues/99394 +repetitiveEval-Ignore from employees | keep emp_no | eval sum = emp_no + 1 | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no From efc2fb299dc02314287c78d534de0debf3f0aa49 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 11 Sep 2023 07:54:38 +0200 Subject: [PATCH 57/69] Tweaked DownsampleClusterDisruptionIT and (#98523) unmuted DownsampleClusterDisruptionIT.testDownsampleIndexWithRollingRestart() and DownsampleClusterDisruptionIT.testDownsampleIndexWithFullClusterRestart() tests Relates to #98485 --- .../xpack/downsample/DownsampleClusterDisruptionIT.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java index b122b59813b43..84b55a5fa8009 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java @@ -209,7 +209,6 @@ public boolean validateClusterForming() { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98485") public void testDownsampleIndexWithRollingRestart() throws Exception { try (InternalTestCluster cluster = internalCluster()) { final List masterNodes = cluster.startMasterOnlyNodes(1); @@ -296,11 +295,10 @@ private void startDownsampleTaskDuringDisruption( } catch (Exception e) { throw new AssertionError(e); } - }, 60, TimeUnit.SECONDS); + }, 120, TimeUnit.SECONDS); disruptionEnd.await(); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98485") public void testDownsampleIndexWithFullClusterRestart() throws Exception { try (InternalTestCluster cluster = internalCluster()) { final List masterNodes = cluster.startMasterOnlyNodes(1); @@ -428,7 +426,7 @@ private void downsample(final String sourceIndex, final String downsampleIndex, assertAcked( internalCluster().client() .execute(DownsampleAction.INSTANCE, new DownsampleAction.Request(sourceIndex, downsampleIndex, TIMEOUT, config)) - .actionGet() + .actionGet(TIMEOUT.millis()) ); } From 2dffc5e57ff6a3d92b3be70f5ceff2589eb3fbda Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 11 Sep 2023 08:31:05 +0100 Subject: [PATCH 58/69] Replace executor(SAME) with DIRECT (#99399) Cleans up the places where we are unnecessarily looking up the SAME executor on a threadpool. --- .../http/netty4/Netty4HttpHeaderThreadContextTests.java | 5 +++-- .../org/elasticsearch/action/bulk/TransportBulkAction.java | 2 +- .../common/util/concurrent/DeterministicTaskQueueTests.java | 6 ++++-- .../elasticsearch/xpack/ml/MlDailyMaintenanceService.java | 3 ++- .../xpack/ml/action/TransportInternalInferModelAction.java | 4 ++-- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderThreadContextTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderThreadContextTests.java index 6b602237199fe..b9b1538b11222 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderThreadContextTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpHeaderThreadContextTests.java @@ -19,6 +19,7 @@ import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpVersion; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.http.netty4.internal.HttpValidator; import org.elasticsearch.test.ESTestCase; @@ -66,7 +67,7 @@ public void testSuccessfulSyncValidationUntamperedThreadContext() throws Excepti channel.pipeline() .addLast( new Netty4HttpHeaderValidator( - getValidator(threadPool.executor(ThreadPool.Names.SAME), isValidationSuccessful, null), + getValidator(EsExecutors.DIRECT_EXECUTOR_SERVICE, isValidationSuccessful, null), threadPool.getThreadContext() ) ); @@ -86,7 +87,7 @@ public void testFailedSyncValidationUntamperedThreadContext() throws Exception { channel.pipeline() .addLast( new Netty4HttpHeaderValidator( - getValidator(threadPool.executor(ThreadPool.Names.SAME), isValidationSuccessful, null), + getValidator(EsExecutors.DIRECT_EXECUTOR_SERVICE, isValidationSuccessful, null), threadPool.getThreadContext() ) ); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 4a3781a5865b8..e0fdc28be0b9a 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -802,7 +802,7 @@ public boolean isForceExecution() { // If a processor went async and returned a response on a different thread then // before we continue the bulk request we should fork back on a write thread: if (originalThread == Thread.currentThread()) { - threadPool.executor(Names.SAME).execute(runnable); + runnable.run(); } else { threadPool.executor(executorName).execute(runnable); } diff --git a/test/framework/src/test/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueueTests.java b/test/framework/src/test/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueueTests.java index fb90b6b87f219..dc51a01da844f 100644 --- a/test/framework/src/test/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueueTests.java +++ b/test/framework/src/test/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueueTests.java @@ -398,9 +398,11 @@ public void testSameExecutor() { final AtomicBoolean executed = new AtomicBoolean(false); final AtomicBoolean executedNested = new AtomicBoolean(false); threadPool.generic().execute(() -> { - threadPool.executor(ThreadPool.Names.SAME).execute(() -> executedNested.set(true)); + final var executor = threadPool.executor(ThreadPool.Names.SAME); + assertSame(EsExecutors.DIRECT_EXECUTOR_SERVICE, executor); + executor.execute(() -> assertTrue(executedNested.compareAndSet(false, true))); assertThat(executedNested.get(), is(true)); - executed.set(true); + assertTrue(executed.compareAndSet(false, true)); }); taskQueue.runAllRunnableTasks(); assertThat(executed.get(), is(true)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 2a9b133cb11dd..7a60dff726909 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Releasable; @@ -289,7 +290,7 @@ public void triggerDeleteJobsInStateDeletingWithoutDeletionTask(ActionListener> chainTaskExecutor = new TypedChainTaskExecutor<>( - threadPool.executor(ThreadPool.Names.SAME), + EsExecutors.DIRECT_EXECUTOR_SERVICE, unused -> true, unused -> true ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java index 43307b05f63f7..17db1cbc5a234 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.Randomness; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Tuple; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseUtils; @@ -23,7 +24,6 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; @@ -171,7 +171,7 @@ private void getModelAndInfer( ) { ActionListener getModelListener = ActionListener.wrap(model -> { TypedChainTaskExecutor typedChainTaskExecutor = new TypedChainTaskExecutor<>( - client.threadPool().executor(ThreadPool.Names.SAME), + EsExecutors.DIRECT_EXECUTOR_SERVICE, // run through all tasks r -> true, // Always fail immediately and return an error From 9f83169676f14de737cd6b45f7d55031d6f80dcf Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 11 Sep 2023 08:58:48 +0100 Subject: [PATCH 59/69] AwaitsFix for #99403 --- .../action/ingest/WriteableIngestDocumentTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java index 784b879e31821..593cfa92877ba 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java @@ -45,6 +45,7 @@ public class WriteableIngestDocumentTests extends AbstractXContentTestCase { + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99403") public void testEqualsAndHashcode() throws Exception { Map sourceAndMetadata = RandomDocumentPicks.randomSource(random()); int numFields = randomIntBetween(1, IngestDocument.Metadata.values().length); From 2ef7da53e3461a6c09484892833a383a2fc3a558 Mon Sep 17 00:00:00 2001 From: Abdon Pijpelink Date: Mon, 11 Sep 2023 10:13:24 +0200 Subject: [PATCH 60/69] [DOCS] Set up redirect for old anchor (#99401) --- docs/reference/redirects.asciidoc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index cf360bdfb9de7..8e1023c47b929 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -1922,7 +1922,13 @@ See <> Refer to <> +[float] +[[configure-remote-clusters-dynamic]] +==== Dynamically configure remote clusters + +Refer to <>. + [role="exclude",id="remote-clusters-privileges"] === Configure roles and users for remote clusters -Refer to <> +Refer to <>. From 7e3391910597387042cf1b684a1994db3967984a Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 11 Sep 2023 09:47:27 +0100 Subject: [PATCH 61/69] Mute continuous transform test cases (#99407) Mutes `HistogramGroupByIT` and `TermsGroupByIT` For https://github.com/elastic/elasticsearch/issues/97263 --- .../transform/integration/continuous/HistogramGroupByIT.java | 2 ++ .../xpack/transform/integration/continuous/TermsGroupByIT.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java index 797d592ef4571..ccf9409d84bd8 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.transform.integration.continuous; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Strings; @@ -28,6 +29,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/97263") public class HistogramGroupByIT extends ContinuousTestCase { private static final String NAME = "continuous-histogram-pivot-test"; diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java index f4b717a108762..8ea0d5e62c6d3 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.transform.integration.continuous; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Strings; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -25,6 +26,7 @@ import static org.hamcrest.Matchers.equalTo; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/97263") public class TermsGroupByIT extends ContinuousTestCase { private static final String NAME = "continuous-terms-pivot-test"; From 356a5401a4c7a7e73cf44b2671ceec85178c103b Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 11 Sep 2023 09:54:04 +0100 Subject: [PATCH 62/69] Deeper `ChunkedBroadcastResponse` chunking (#99364) Today the chunked responses to broadcast APIs (indices stats, indices segments and field usage stats) emit one chunk per index, but each index-level chunk could itself scale with the number of shard copies, segments, or fields in the index. This commit introduces finer-grained chunking, such that each chunk now has O(1) size. --- .../segments/IndicesSegmentResponse.java | 188 ++++++++++-------- .../stats/FieldUsageStatsResponse.java | 19 +- .../indices/stats/IndicesStatsResponse.java | 98 +++++---- .../segments/IndicesSegmentResponseTests.java | 2 +- .../stats/FieldUsageStatsResponseTests.java | 2 +- .../stats/IndicesStatsResponseTests.java | 5 +- 6 files changed, 181 insertions(+), 133 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java index 9b1eb1baf197c..06d8eb5d99152 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.admin.indices.segments; import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; import org.elasticsearch.action.support.DefaultShardOperationFailedException; @@ -18,13 +17,14 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.index.engine.Segment; import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -79,89 +79,115 @@ public void writeTo(StreamOutput out) throws IOException { @Override protected Iterator customXContentChunks(ToXContent.Params params) { return Iterators.concat( - Iterators.single((builder, p) -> builder.startObject(Fields.INDICES)), - Iterators.map(getIndices().values().iterator(), indexSegments -> (builder, p) -> { - builder.startObject(indexSegments.getIndex()); - - builder.startObject(Fields.SHARDS); - for (IndexShardSegments indexSegment : indexSegments) { - builder.startArray(Integer.toString(indexSegment.shardId().id())); - for (ShardSegments shardSegments : indexSegment) { - builder.startObject(); - - builder.startObject(Fields.ROUTING); - builder.field(Fields.STATE, shardSegments.getShardRouting().state()); - builder.field(Fields.PRIMARY, shardSegments.getShardRouting().primary()); - builder.field(Fields.NODE, shardSegments.getShardRouting().currentNodeId()); - if (shardSegments.getShardRouting().relocatingNodeId() != null) { - builder.field(Fields.RELOCATING_NODE, shardSegments.getShardRouting().relocatingNodeId()); - } - builder.endObject(); - - builder.field(Fields.NUM_COMMITTED_SEGMENTS, shardSegments.getNumberOfCommitted()); - builder.field(Fields.NUM_SEARCH_SEGMENTS, shardSegments.getNumberOfSearch()); - - builder.startObject(Fields.SEGMENTS); - for (Segment segment : shardSegments) { - builder.startObject(segment.getName()); - builder.field(Fields.GENERATION, segment.getGeneration()); - builder.field(Fields.NUM_DOCS, segment.getNumDocs()); - builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs()); - builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, segment.getSize()); - if (builder.getRestApiVersion() == RestApiVersion.V_7) { - builder.humanReadableField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, ByteSizeValue.ZERO); - } - builder.field(Fields.COMMITTED, segment.isCommitted()); - builder.field(Fields.SEARCH, segment.isSearch()); - if (segment.getVersion() != null) { - builder.field(Fields.VERSION, segment.getVersion()); - } - if (segment.isCompound() != null) { - builder.field(Fields.COMPOUND, segment.isCompound()); - } - if (segment.getMergeId() != null) { - builder.field(Fields.MERGE_ID, segment.getMergeId()); - } - if (segment.getSegmentSort() != null) { - toXContent(builder, segment.getSegmentSort()); - } - if (segment.attributes != null && segment.attributes.isEmpty() == false) { - builder.field("attributes", segment.attributes); - } - builder.endObject(); - } - builder.endObject(); - - builder.endObject(); - } - builder.endArray(); - } - builder.endObject(); - builder.endObject(); - return builder; - }), - Iterators.single((builder, p) -> builder.endObject()) + ChunkedToXContentHelper.startObject(Fields.INDICES), + Iterators.flatMap( + getIndices().values().iterator(), + indexSegments -> Iterators.concat( + + ChunkedToXContentHelper.singleChunk( + (builder, p) -> builder.startObject(indexSegments.getIndex()).startObject(Fields.SHARDS) + ), + Iterators.flatMap( + indexSegments.iterator(), + indexSegment -> Iterators.concat( + + ChunkedToXContentHelper.startArray(Integer.toString(indexSegment.shardId().id())), + Iterators.flatMap( + indexSegment.iterator(), + shardSegments -> Iterators.concat( + + ChunkedToXContentHelper.singleChunk((builder, p) -> { + builder.startObject(); + + builder.startObject(Fields.ROUTING); + builder.field(Fields.STATE, shardSegments.getShardRouting().state()); + builder.field(Fields.PRIMARY, shardSegments.getShardRouting().primary()); + builder.field(Fields.NODE, shardSegments.getShardRouting().currentNodeId()); + if (shardSegments.getShardRouting().relocatingNodeId() != null) { + builder.field(Fields.RELOCATING_NODE, shardSegments.getShardRouting().relocatingNodeId()); + } + builder.endObject(); + + builder.field(Fields.NUM_COMMITTED_SEGMENTS, shardSegments.getNumberOfCommitted()); + builder.field(Fields.NUM_SEARCH_SEGMENTS, shardSegments.getNumberOfSearch()); + + builder.startObject(Fields.SEGMENTS); + return builder; + }), + Iterators.flatMap( + shardSegments.iterator(), + segment -> Iterators.concat( + + ChunkedToXContentHelper.singleChunk((builder, p) -> { + builder.startObject(segment.getName()); + builder.field(Fields.GENERATION, segment.getGeneration()); + builder.field(Fields.NUM_DOCS, segment.getNumDocs()); + builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs()); + builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, segment.getSize()); + if (builder.getRestApiVersion() == RestApiVersion.V_7) { + builder.humanReadableField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, ByteSizeValue.ZERO); + } + builder.field(Fields.COMMITTED, segment.isCommitted()); + builder.field(Fields.SEARCH, segment.isSearch()); + if (segment.getVersion() != null) { + builder.field(Fields.VERSION, segment.getVersion()); + } + if (segment.isCompound() != null) { + builder.field(Fields.COMPOUND, segment.isCompound()); + } + if (segment.getMergeId() != null) { + builder.field(Fields.MERGE_ID, segment.getMergeId()); + } + return builder; + }), + getSegmentSortChunks(segment.getSegmentSort()), + ChunkedToXContentHelper.singleChunk((builder, p) -> { + if (segment.attributes != null && segment.attributes.isEmpty() == false) { + builder.field("attributes", segment.attributes); + } + builder.endObject(); + return builder; + }) + ) + ), + ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endObject().endObject()) + ) + ), + ChunkedToXContentHelper.endArray() + ) + ), + ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endObject().endObject()) + ) + ), + ChunkedToXContentHelper.endObject() ); } - private static void toXContent(XContentBuilder builder, Sort sort) throws IOException { - builder.startArray("sort"); - for (SortField field : sort.getSort()) { - builder.startObject(); - builder.field("field", field.getField()); - if (field instanceof SortedNumericSortField) { - builder.field("mode", ((SortedNumericSortField) field).getSelector().toString().toLowerCase(Locale.ROOT)); - } else if (field instanceof SortedSetSortField) { - builder.field("mode", ((SortedSetSortField) field).getSelector().toString().toLowerCase(Locale.ROOT)); - } - if (field.getMissingValue() != null) { - builder.field("missing", field.getMissingValue().toString()); - } - builder.field("reverse", field.getReverse()); - builder.endObject(); + private static Iterator getSegmentSortChunks(@Nullable Sort segmentSort) { + if (segmentSort == null) { + return Collections.emptyIterator(); } - builder.endArray(); + + return Iterators.concat( + ChunkedToXContentHelper.startArray("sort"), + Iterators.map(Iterators.forArray(segmentSort.getSort()), field -> (builder, p) -> { + builder.startObject(); + builder.field("field", field.getField()); + if (field instanceof SortedNumericSortField sortedNumericSortField) { + builder.field("mode", sortedNumericSortField.getSelector().toString().toLowerCase(Locale.ROOT)); + } else if (field instanceof SortedSetSortField sortedSetSortField) { + builder.field("mode", sortedSetSortField.getSelector().toString().toLowerCase(Locale.ROOT)); + } + if (field.getMissingValue() != null) { + builder.field("missing", field.getMissingValue().toString()); + } + builder.field("reverse", field.getReverse()); + builder.endObject(); + return builder; + }), + ChunkedToXContentHelper.endArray() + ); } static final class Fields { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java index 5a16ecf976354..e32934a5706b2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java @@ -10,8 +10,10 @@ import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.ChunkedBroadcastResponse; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; @@ -50,14 +52,13 @@ public Map> getStats() { @Override protected Iterator customXContentChunks(ToXContent.Params params) { - return stats.entrySet().stream().sorted(Map.Entry.comparingByKey()).map(entry -> (ToXContent) (builder, p) -> { - builder.startObject(entry.getKey()); - builder.startArray("shards"); - for (FieldUsageShardResponse resp : entry.getValue()) { - resp.toXContent(builder, params); - } - builder.endArray(); - return builder.endObject(); - }).iterator(); + return Iterators.flatMap( + stats.entrySet().stream().sorted(Map.Entry.comparingByKey()).iterator(), + entry -> Iterators.concat( + ChunkedToXContentHelper.singleChunk((builder, p) -> builder.startObject(entry.getKey()).startArray("shards")), + entry.getValue().iterator(), + ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endArray().endObject()) + ) + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java index 70959babf312e..1ea0820492b04 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java @@ -23,11 +23,13 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.index.Index; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -181,46 +183,66 @@ public void writeTo(StreamOutput out) throws IOException { protected Iterator customXContentChunks(ToXContent.Params params) { final ClusterStatsLevel level = ClusterStatsLevel.of(params, ClusterStatsLevel.INDICES); if (level == ClusterStatsLevel.INDICES || level == ClusterStatsLevel.SHARDS) { - return Iterators.concat(Iterators.single(((builder, p) -> { - commonStats(builder, p); - return builder.startObject(Fields.INDICES); - })), Iterators.map(getIndices().values().iterator(), indexStats -> (builder, p) -> { - builder.startObject(indexStats.getIndex()); - builder.field("uuid", indexStats.getUuid()); - if (indexStats.getHealth() != null) { - builder.field("health", indexStats.getHealth().toString().toLowerCase(Locale.ROOT)); - } - if (indexStats.getState() != null) { - builder.field("status", indexStats.getState().toString().toLowerCase(Locale.ROOT)); - } - builder.startObject("primaries"); - indexStats.getPrimaries().toXContent(builder, p); - builder.endObject(); - - builder.startObject("total"); - indexStats.getTotal().toXContent(builder, p); - builder.endObject(); - - if (level == ClusterStatsLevel.SHARDS) { - builder.startObject(Fields.SHARDS); - for (IndexShardStats indexShardStats : indexStats) { - builder.startArray(Integer.toString(indexShardStats.getShardId().id())); - for (ShardStats shardStats : indexShardStats) { - builder.startObject(); - shardStats.toXContent(builder, p); + return Iterators.concat( + + ChunkedToXContentHelper.singleChunk((builder, p) -> { + commonStats(builder, p); + return builder.startObject(Fields.INDICES); + }), + Iterators.flatMap( + getIndices().values().iterator(), + indexStats -> Iterators.concat( + + ChunkedToXContentHelper.singleChunk((builder, p) -> { + builder.startObject(indexStats.getIndex()); + builder.field("uuid", indexStats.getUuid()); + if (indexStats.getHealth() != null) { + builder.field("health", indexStats.getHealth().toString().toLowerCase(Locale.ROOT)); + } + if (indexStats.getState() != null) { + builder.field("status", indexStats.getState().toString().toLowerCase(Locale.ROOT)); + } + builder.startObject("primaries"); + indexStats.getPrimaries().toXContent(builder, p); + builder.endObject(); + + builder.startObject("total"); + indexStats.getTotal().toXContent(builder, p); builder.endObject(); - } - builder.endArray(); - } - builder.endObject(); - } - return builder.endObject(); - }), Iterators.single((b, p) -> b.endObject())); + return builder; + }), + + level == ClusterStatsLevel.SHARDS + ? Iterators.concat( + ChunkedToXContentHelper.startObject(Fields.SHARDS), + Iterators.flatMap( + indexStats.iterator(), + indexShardStats -> Iterators.concat( + ChunkedToXContentHelper.startArray(Integer.toString(indexShardStats.getShardId().id())), + Iterators.map(indexShardStats.iterator(), shardStats -> (builder, p) -> { + builder.startObject(); + shardStats.toXContent(builder, p); + builder.endObject(); + return builder; + }), + ChunkedToXContentHelper.endArray() + ) + ), + ChunkedToXContentHelper.endObject() + ) + : Collections.emptyIterator(), + + ChunkedToXContentHelper.endObject() + ) + ), + ChunkedToXContentHelper.endObject() + ); + } else { + return ChunkedToXContentHelper.singleChunk((builder, p) -> { + commonStats(builder, p); + return builder; + }); } - return Iterators.single((b, p) -> { - commonStats(b, p); - return b; - }); } private void commonStats(XContentBuilder builder, ToXContent.Params p) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java index 5fe7a528d68c7..ad7689320a94f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java @@ -67,7 +67,7 @@ public void testChunking() { 0, Collections.emptyList() ), - response -> response.getIndices().size() + 4 + response -> 11 * response.getIndices().size() + 4 ); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java index 6fb53760de5c2..32bedb6082842 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java @@ -46,7 +46,7 @@ public void testToXContentChunkPerIndex() { AbstractChunkedSerializingTestCase.assertChunkCount( new FieldUsageStatsResponse(indices, indices, 0, List.of(), perIndex), - ignored -> indices + 2 + ignored -> 3 * indices + 2 ); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java index a1280f7658804..6ee17ec7963b6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.json.JsonXContent; -import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; @@ -116,7 +115,7 @@ public void testGetIndices() { } } - public void testChunkedEncodingPerIndex() throws IOException { + public void testChunkedEncodingPerIndex() { final int shards = randomIntBetween(1, 10); final List stats = new ArrayList<>(shards); for (int i = 0; i < shards; i++) { @@ -143,7 +142,7 @@ public void testChunkedEncodingPerIndex() throws IOException { AbstractChunkedSerializingTestCase.assertChunkCount( indicesStatsResponse, new ToXContent.MapParams(Map.of("level", "indices")), - ignored -> 4 + shards + ignored -> 4 + 2 * shards ); } From c0a2a3442d80747ab7fa8bddeaafe45bda1a0039 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 11 Sep 2023 11:09:11 +0200 Subject: [PATCH 63/69] Make ESQL RepetitiveEval test deterministic (#99405) --- .../esql/qa/testFixtures/src/main/resources/eval.csv-spec | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec index 11111db4620d9..9108b0d6eedb6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/eval.csv-spec @@ -110,9 +110,8 @@ true | false | false | true ; -// AwaitsFix https://github.com/elastic/elasticsearch/issues/99394 -repetitiveEval-Ignore -from employees | keep emp_no | eval sum = emp_no + 1 +repetitiveEval +from employees | sort emp_no | keep emp_no | eval sum = emp_no + 1 | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no @@ -160,7 +159,6 @@ from employees | keep emp_no | eval sum = emp_no + 1 | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | eval sum = sum + emp_no | limit 3 -| sort emp_no ; emp_no:i | sum:i From 871ca07e99b4cbb7ca7bff2e57864739c28c3556 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 11 Sep 2023 10:39:55 +0100 Subject: [PATCH 64/69] Mute SearchWithRandomIOExceptionsIT::testRandomDirectoryIOExceptions (#99410) For #99174 --- .../search/basic/SearchWithRandomIOExceptionsIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java index 804ad3cdfd523..e829c6da01b26 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java @@ -43,6 +43,7 @@ protected Collection> nodePlugins() { return Arrays.asList(MockFSIndexStore.TestPlugin.class); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99174") public void testRandomDirectoryIOExceptions() throws IOException, InterruptedException, ExecutionException { String mapping = Strings.toString( XContentFactory.jsonBuilder() From af4b2d98be9cc4f153e8ca716356f80c47009917 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 11 Sep 2023 10:47:20 +0100 Subject: [PATCH 65/69] Mute MoreExpressionIT::testStringSpecialValueVariable (#99411) For #99156 --- .../org/elasticsearch/script/expression/MoreExpressionIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java index e7d6d127174ec..02c17977daf6a 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java @@ -502,6 +502,7 @@ public void testSpecialValueVariable() throws Exception { assertThat(stats.getAvg(), equalTo(3.0)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99156") public void testStringSpecialValueVariable() throws Exception { // i.e. expression script for term aggregations, which is not allowed assertAcked(indicesAdmin().prepareCreate("test").setMapping("text", "type=keyword").get()); From 3ca48a47526ac2d1df7146956f7d770102aaa57e Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 11 Sep 2023 11:50:23 +0200 Subject: [PATCH 66/69] NamedWritable keys must be strings, not dynamic (#99408) When developing the GenericNamedWriteable instances we were dynamically generating the key names from the class names, but it is better to use strings so that class name refactoring does not change the keys causing compatibility issues in inter-node communication in mixed clusters. --- .../src/main/java/org/elasticsearch/search/SearchModule.java | 4 +--- .../java/org/elasticsearch/xpack/spatial/SpatialPlugin.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index babce70239871..111bec2c88509 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -317,9 +317,7 @@ public SearchModule(Settings settings, List plugins) { registerIntervalsSourceProviders(); requestCacheKeyDifferentiator = registerRequestCacheKeyDifferentiator(plugins); namedWriteables.addAll(SortValue.namedWriteables()); - registerGenericNamedWriteable( - new SearchPlugin.GenericNamedWriteableSpec(GeoBoundingBox.class.getSimpleName(), GeoBoundingBox::new) - ); + registerGenericNamedWriteable(new SearchPlugin.GenericNamedWriteableSpec("GeoBoundingBox", GeoBoundingBox::new)); } public List getNamedWriteables() { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index 22f8ae8969ec7..66281cd21856b 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -194,7 +194,7 @@ public Map getProcessors(Processor.Parameters paramet @Override public List getGenericNamedWriteables() { - return List.of(new GenericNamedWriteableSpec(CartesianBoundingBox.class.getSimpleName(), CartesianBoundingBox::new)); + return List.of(new GenericNamedWriteableSpec("CartesianBoundingBox", CartesianBoundingBox::new)); } private static void registerGeoShapeBoundsAggregator(ValuesSourceRegistry.Builder builder) { From 957989474ee40d1730a7e574460e5f71ee5c2f24 Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Reddy Burri Date: Mon, 11 Sep 2023 16:05:20 +0530 Subject: [PATCH 67/69] Add "manage" permission for fleet managed threat intel indices (#99231) * Add "manage" permission needed for creating and maintaining aliases of threat intel transforms --- docs/changelog/99231.yaml | 5 +++++ .../authz/store/KibanaOwnedReservedRoleDescriptors.java | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/99231.yaml diff --git a/docs/changelog/99231.yaml b/docs/changelog/99231.yaml new file mode 100644 index 0000000000000..9f5dfa1137587 --- /dev/null +++ b/docs/changelog/99231.yaml @@ -0,0 +1,5 @@ +pr: 99231 +summary: Add manage permission for fleet managed threat intel indices +area: Authorization +type: enhancement +issues: [] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 0245e559496c8..18c72ea8d42cb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -281,13 +281,12 @@ static RoleDescriptor kibanaSystem(String name) { RoleDescriptor.IndicesPrivileges.builder() .indices("logs-ti_*_latest.*") .privileges( - // Require "create_index", "delete_index", "read", "index", "delete", IndicesAliasesAction.NAME, and - // UpdateSettingsAction.NAME for transform "create_index", "delete_index", "read", "index", "delete", + "manage", IndicesAliasesAction.NAME, UpdateSettingsAction.NAME ) From 4aa0c698f2f7eff5550ef4fcd111703dfd40050b Mon Sep 17 00:00:00 2001 From: Johannes Mahne Date: Mon, 11 Sep 2023 12:56:34 +0200 Subject: [PATCH 68/69] Update ilm-shrink.asciidoc (#99366) Formatting --- docs/reference/ilm/actions/ilm-shrink.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ilm/actions/ilm-shrink.asciidoc b/docs/reference/ilm/actions/ilm-shrink.asciidoc index 727124a18ea43..d440943c2ee75 100644 --- a/docs/reference/ilm/actions/ilm-shrink.asciidoc +++ b/docs/reference/ilm/actions/ilm-shrink.asciidoc @@ -17,7 +17,7 @@ stream. You cannot perform the `shrink` action on a write index. To use the `shrink` action in the `hot` phase, the `rollover` action *must* be present. If no rollover action is configured, {ilm-init} will reject the policy. -The shrink action will unset the index's index.routing.allocation.total_shards_per_node +The shrink action will unset the index's `index.routing.allocation.total_shards_per_node` setting, meaning that there will be no limit. This is to ensure that all shards of the index can be copied to a single node. This setting change will persist on the index even after the step completes. From 19378f656e04a578f28c04c6812de702508c74eb Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 11 Sep 2023 13:05:40 +0200 Subject: [PATCH 69/69] Mark 'index.hidden' setting as public (#99338) the setting is used by both external and internal users at the moment. this commit is marking `index.hidden` setting as public for serverless --- .../java/org/elasticsearch/cluster/metadata/IndexMetadata.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 517bc2293ab6b..ab67478192c11 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -475,7 +475,8 @@ public Iterator> settings() { SETTING_INDEX_HIDDEN, false, Property.Dynamic, - Property.IndexScope + Property.IndexScope, + Property.ServerlessPublic ); /**