From bf34036c8c6bf7a56c814bbfbb82e768547fdfba Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 21 Sep 2023 13:00:13 +0100 Subject: [PATCH 01/16] Discovery troubleshooting next steps (#99743) Adds a little more detail on how to react if you see evidence that the Elasticsearch process is pausing for a long time due to long GCs or VM pauses. --- .../troubleshooting/network-timeouts.asciidoc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/reference/troubleshooting/network-timeouts.asciidoc b/docs/reference/troubleshooting/network-timeouts.asciidoc index df961c83a292d..c15c5ee0d58a5 100644 --- a/docs/reference/troubleshooting/network-timeouts.asciidoc +++ b/docs/reference/troubleshooting/network-timeouts.asciidoc @@ -1,11 +1,17 @@ tag::troubleshooting-network-timeouts-gc-vm[] * GC pauses are recorded in the GC logs that {es} emits by default, and also usually by the `JvmMonitorService` in the main node logs. Use these logs to -confirm whether or not GC is resulting in delays. +confirm whether or not the node is experiencing high heap usage with long GC +pauses. If so, <> has some suggestions for further investigation but typically you +will need to capture a heap dump during a time of high heap usage to fully +understand the problem. * VM pauses also affect other processes on the same host. A VM pause also typically causes a discontinuity in the system clock, which {es} will report in -its logs. +its logs. If you see evidence of other processes pausing at the same time, or +unexpected clock discontinuities, investigate the infrastructure on which you +are running {es}. end::troubleshooting-network-timeouts-gc-vm[] tag::troubleshooting-network-timeouts-packet-capture-elections[] From ee133e0758362bd3f13428067c549e2ec220652d Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 21 Sep 2023 09:03:25 -0400 Subject: [PATCH 02/16] Bump versions after 8.10.2 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 16 ++++++++++++++++ .buildkite/pipelines/periodic.yml | 10 ++++++++++ .ci/bwcVersions | 1 + .ci/snapshotBwcVersions | 2 +- .../src/main/java/org/elasticsearch/Version.java | 1 + 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 0f89ba9b8aeba..07cf55e0b3adf 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -40,7 +40,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.14", "8.10.2", "8.11.0"] + BWC_VERSION: ["7.17.14", "8.10.3", "8.11.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 1fd69ec8d8a2c..ce0746a5726cc 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -1632,6 +1632,22 @@ steps: env: BWC_VERSION: 8.10.2 + - label: "{{matrix.image}} / 8.10.3 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.3 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.10.3 + - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0 timeout_in_minutes: 300 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index fe28b0a0cea64..967dcbb8cf535 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -1002,6 +1002,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 8.10.2 + - label: 8.10.3 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.3#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.10.3 - label: 8.11.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest timeout_in_minutes: 300 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index ea42d7d4f0cc6..90ba9b8131dcd 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -99,4 +99,5 @@ BWC_VERSION: - "8.10.0" - "8.10.1" - "8.10.2" + - "8.10.3" - "8.11.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 2620f98f1cec1..1bc1fa321d9da 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,4 +1,4 @@ BWC_VERSION: - "7.17.14" - - "8.10.2" + - "8.10.3" - "8.11.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 43b4d2c8bb37a..591f93a34726f 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -150,6 +150,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_10_0 = new Version(8_10_00_99); public static final Version V_8_10_1 = new Version(8_10_01_99); public static final Version V_8_10_2 = new Version(8_10_02_99); + public static final Version V_8_10_3 = new Version(8_10_03_99); public static final Version V_8_11_0 = new Version(8_11_00_99); public static final Version CURRENT = V_8_11_0; From 010b11afc0d347f920c0492924eb6e0f0e1d08c0 Mon Sep 17 00:00:00 2001 From: Marco Liberati Date: Thu, 21 Sep 2023 15:09:03 +0200 Subject: [PATCH 03/16] [ES|QL] Improve error message in case of unsupported time interval (#99662) * Improve error message in case of unsupported time interval * From feedback: improve message Co-authored-by: Bogdan Pintea * From feedback: update test * Typo --------- Co-authored-by: Bogdan Pintea --- .../org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java | 2 +- .../org/elasticsearch/xpack/esql/parser/ExpressionTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index bbffa6c63e9b1..aa653d36d141b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -233,7 +233,7 @@ public Object visitQualifiedIntegerLiteral(EsqlBaseParser.QualifiedIntegerLitera case "month", "months" -> Period.ofMonths(safeToInt(safeToLong(value))); case "year", "years" -> Period.ofYears(safeToInt(safeToLong(value))); - default -> throw new ParsingException(source, "Unexpected numeric qualifier '{}'", qualifier); + default -> throw new ParsingException(source, "Unexpected time interval qualifier: '{}'", qualifier); }; return new Literal(source, quantity, quantity instanceof Duration ? TIME_DURATION : DATE_PERIOD); } catch (QlIllegalArgumentException | ArithmeticException e) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java index ae5de9a0a13e2..d3067ba7f306f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java @@ -410,7 +410,7 @@ public void testDatePeriodLiterals() { } public void testUnknownNumericQualifier() { - assertParsingException(() -> whereExpression("1 decade"), "Unexpected numeric qualifier 'decade'"); + assertParsingException(() -> whereExpression("1 decade"), "Unexpected time interval qualifier: 'decade'"); } public void testQualifiedDecimalLiteral() { From 61abe140c79bb69884f4801291e393e3b501d07d Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 21 Sep 2023 06:37:26 -0700 Subject: [PATCH 04/16] Upgrade bundled JDK to Java 21 (#99724) Java 21 is now GA making Java 20 EOL. --- build-tools-internal/version.properties | 2 +- docs/changelog/99724.yaml | 5 +++++ gradle/verification-metadata.xml | 21 +++++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/99724.yaml diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 274f6a98dc464..aebdabee7aed7 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -2,7 +2,7 @@ elasticsearch = 8.11.0 lucene = 9.7.0 bundled_jdk_vendor = openjdk -bundled_jdk = 20.0.2+9@6e380f22cbe7469fa75fb448bd903d8e +bundled_jdk = 21+35@fd2272bbf8e04c3dbaee13770090416c # optional dependencies spatial4j = 0.7 diff --git a/docs/changelog/99724.yaml b/docs/changelog/99724.yaml new file mode 100644 index 0000000000000..4fe78687bf72b --- /dev/null +++ b/docs/changelog/99724.yaml @@ -0,0 +1,5 @@ +pr: 99724 +summary: Upgrade bundled JDK to Java 21 +area: Packaging +type: upgrade +issues: [] diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 6cd8d56be9530..aec85d9ed5d6b 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1600,6 +1600,27 @@ + + + + + + + + + + + + + + + + + + + + + From 06f09d861de538c6989186187b2fa8d96e5e12fe Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 21 Sep 2023 14:42:04 +0100 Subject: [PATCH 05/16] Revert "Migrate rolling upgrade tests to new junit format" (#99750) Reverts elastic/elasticsearch#99572 and #99733 The new tests are unstable, and don't work on CI. This re-opens https://github.com/elastic/elasticsearch/issues/97200 --- .../internal/RestrictedBuildApiService.java | 1 - .../resources/checkstyle_suppressions.xml | 2 +- qa/full-cluster-restart/build.gradle | 5 - qa/rolling-upgrade-legacy/build.gradle | 143 ------- qa/rolling-upgrade/build.gradle | 137 ++++++- .../ParameterizedRollingUpgradeTestCase.java | 226 ----------- .../UpgradeWithOldIndexSettingsIT.java | 132 ------- .../upgrades/AbstractRollingTestCase.java | 18 + .../upgrades/DesiredNodesUpgradeIT.java | 66 ++-- .../upgrades/FeatureUpgradeIT.java | 16 +- .../elasticsearch/upgrades/FieldCapsIT.java | 34 +- .../elasticsearch/upgrades/IndexingIT.java | 363 ++++++++++-------- .../elasticsearch/upgrades/RecoveryIT.java | 20 +- .../upgrades/SnapshotBasedRecoveryIT.java | 160 ++++---- .../upgrades/SystemIndicesUpgradeIT.java | 12 +- .../org/elasticsearch/upgrades/TsdbIT.java | 46 +-- .../UpgradeClusterClientYamlTestSuiteIT.java | 5 + .../UpgradeWithOldIndexSettingsIT.java | 131 +++++++ .../org/elasticsearch/upgrades/XPackIT.java | 14 +- .../test/mixed_cluster/10_basic.yml | 0 .../mixed_cluster/20_camel_case_on_format.yml | 0 .../test/mixed_cluster/30_vector_search.yml | 0 .../test/old_cluster/10_basic.yml | 0 .../old_cluster/20_camel_case_on_format.yml | 0 .../test/old_cluster/30_vector_search.yml | 2 + .../test/upgraded_cluster/10_basic.yml | 0 .../20_camel_case_on_format.yml | 0 .../upgraded_cluster/30_vector_search.yml | 0 28 files changed, 636 insertions(+), 897 deletions(-) delete mode 100644 qa/rolling-upgrade-legacy/build.gradle delete mode 100644 qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java delete mode 100644 qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java (79%) rename qa/rolling-upgrade/src/{javaRestTest => test}/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java (82%) rename qa/rolling-upgrade/src/{javaRestTest => test}/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java (90%) rename qa/rolling-upgrade/src/{javaRestTest => test}/java/org/elasticsearch/upgrades/FieldCapsIT.java (94%) rename qa/rolling-upgrade/src/{javaRestTest => test}/java/org/elasticsearch/upgrades/IndexingIT.java (52%) rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java (97%) rename qa/rolling-upgrade/src/{javaRestTest => test}/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java (61%) rename qa/rolling-upgrade/src/{javaRestTest => test}/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java (95%) rename qa/rolling-upgrade/src/{javaRestTest => test}/java/org/elasticsearch/upgrades/TsdbIT.java (90%) rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java (95%) create mode 100644 qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java rename qa/rolling-upgrade/src/{javaRestTest => test}/java/org/elasticsearch/upgrades/XPackIT.java (93%) rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml (100%) rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/resources/rest-api-spec/test/mixed_cluster/20_camel_case_on_format.yml (100%) rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/resources/rest-api-spec/test/mixed_cluster/30_vector_search.yml (100%) rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml (100%) rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/resources/rest-api-spec/test/old_cluster/20_camel_case_on_format.yml (100%) rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml (99%) rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml (100%) rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/resources/rest-api-spec/test/upgraded_cluster/20_camel_case_on_format.yml (100%) rename qa/{rolling-upgrade-legacy => rolling-upgrade}/src/test/resources/rest-api-spec/test/upgraded_cluster/30_vector_search.yml (100%) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java index 088f8290e713f..e9438eabadbb6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java @@ -67,7 +67,6 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters"); map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version"); map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade"); - map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy"); map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-http"); map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-disabled"); map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies"); diff --git a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml index 9f074513b6d4e..211faf973b772 100644 --- a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml +++ b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml @@ -32,7 +32,7 @@ - + diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle index b6ecc054c9e41..06442b7cfe6a6 100644 --- a/qa/full-cluster-restart/build.gradle +++ b/qa/full-cluster-restart/build.gradle @@ -13,11 +13,6 @@ apply plugin: 'elasticsearch.internal-java-rest-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.bwc-test' -test { - // CI doesn't like it when there's multiple clusters running at once - maxParallelForks = 1 -} - BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) diff --git a/qa/rolling-upgrade-legacy/build.gradle b/qa/rolling-upgrade-legacy/build.gradle deleted file mode 100644 index 7aca34bef8a1b..0000000000000 --- a/qa/rolling-upgrade-legacy/build.gradle +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - - -import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.internal.BwcVersions -import org.elasticsearch.gradle.internal.info.BuildParams -import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask - -apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.standalone-rest-test' -apply plugin: 'elasticsearch.bwc-test' -apply plugin: 'elasticsearch.rest-resources' - -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> - /* - * NOTE: This module is for the tests that were problematic when converting :qa:rolling-upgrade to the junit-based bwc test definition - * Over time, these should be migrated into the :qa:rolling-upgrade module and fixed properly - * - * The goal here is to: - *
    - *
  • start three nodes on the old version - *
  • run tests with systemProperty 'tests.rest.suite', 'old_cluster' - *
  • upgrade one node - *
  • run tests with systemProperty 'tests.rest.suite', 'mixed_cluster' - *
  • upgrade one more node - *
  • run tests with systemProperty 'tests.rest.suite', 'mixed_cluster' again - *
  • updgrade the last node - *
  • run tests with systemProperty 'tests.rest.suite', 'upgraded_cluster' - *
- */ - - def baseCluster = testClusters.register(baseName) { - versions = [bwcVersion.toString(), project.version] - numberOfNodes = 3 - - setting 'repositories.url.allowed_urls', 'http://snapshot.test*' - setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" - setting 'xpack.security.enabled', 'false' - setting 'logger.org.elasticsearch.cluster.service.MasterService', 'TRACE' - setting 'logger.org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator', 'TRACE' - setting 'logger.org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders', 'TRACE' - requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") - } - - String oldVersion = bwcVersion.toString() - - tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { - dependsOn "processTestResources" - useCluster baseCluster - mustRunAfter("precommit") - doFirst { - delete("${buildDir}/cluster/shared/repo/${baseName}") - } - def excludeList = [] - systemProperty 'tests.rest.suite', 'old_cluster' - systemProperty 'tests.upgrade_from_version', oldVersion - nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) - nonInputProperties.systemProperty('tests.clustername', baseName) - if (bwcVersion.before("8.4.0")) { - excludeList.addAll(["old_cluster/30_vector_search/*"]) - } else if (bwcVersion.before("8.6.0")) { - excludeList.addAll(["old_cluster/30_vector_search/Create indexed byte vectors and search"]) - } - if (excludeList.isEmpty() == false) { - systemProperty 'tests.rest.blacklist', excludeList.join(',') - } - } - - tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) { - dependsOn "${baseName}#oldClusterTest" - useCluster baseCluster - doFirst { - baseCluster.get().nextNodeToNextVersion() - } - systemProperty 'tests.rest.suite', 'mixed_cluster' - systemProperty 'tests.upgrade_from_version', oldVersion - systemProperty 'tests.first_round', 'true' - nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) - nonInputProperties.systemProperty('tests.clustername', baseName) - def excludeList = [] - if (bwcVersion.before("8.4.0")) { - excludeList.addAll(["mixed_cluster/30_vector_search/*"]) - } else if (bwcVersion.before("8.6.0")) { - excludeList.addAll(["mixed_cluster/30_vector_search/Search byte indices created in old cluster"]) - } - if (excludeList.isEmpty() == false) { - systemProperty 'tests.rest.blacklist', excludeList.join(',') - } - } - - tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) { - dependsOn "${baseName}#oneThirdUpgradedTest" - useCluster baseCluster - doFirst { - baseCluster.get().nextNodeToNextVersion() - } - systemProperty 'tests.rest.suite', 'mixed_cluster' - systemProperty 'tests.upgrade_from_version', oldVersion - systemProperty 'tests.first_round', 'false' - nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) - nonInputProperties.systemProperty('tests.clustername', baseName) - def excludeList = [] - if (bwcVersion.before("8.4.0")) { - excludeList.addAll(["mixed_cluster/30_vector_search/*"]) - } else if (bwcVersion.before("8.6.0")) { - excludeList.addAll(["mixed_cluster/30_vector_search/Search byte indices created in old cluster"]) - } - if (excludeList.isEmpty() == false) { - systemProperty 'tests.rest.blacklist', excludeList.join(',') - } - } - - tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { - dependsOn "${baseName}#twoThirdsUpgradedTest" - doFirst { - baseCluster.get().nextNodeToNextVersion() - } - useCluster testClusters.named(baseName) - systemProperty 'tests.rest.suite', 'upgraded_cluster' - systemProperty 'tests.upgrade_from_version', oldVersion - nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) - nonInputProperties.systemProperty('tests.clustername', baseName) - def excludeList = [] - if (bwcVersion.before("8.4.0")) { - excludeList.addAll(["upgraded_cluster/30_vector_search/*"]) - } else if (bwcVersion.before("8.6.0")) { - excludeList.addAll(["upgraded_cluster/30_vector_search/Search byte indices created in old cluster"]) - } - if (excludeList.isEmpty() == false) { - systemProperty 'tests.rest.blacklist', excludeList.join(',') - } - } - - tasks.register(bwcTaskName(bwcVersion)) { - dependsOn tasks.named("${baseName}#upgradedClusterTest") - } -} diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 9abe03ab923b1..d3078dd8c9381 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -6,30 +6,135 @@ * Side Public License, v 1. */ + +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.internal.BwcVersions import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.internal-testclusters' -apply plugin: 'elasticsearch.internal-java-rest-test' -apply plugin: 'elasticsearch.internal-test-artifact-base' +apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.bwc-test' +apply plugin: 'elasticsearch.rest-resources' -testArtifacts { - registerTestArtifactFromSourceSet(sourceSets.javaRestTest) -} +BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> + /* + * The goal here is to: + *
    + *
  • start three nodes on the old version + *
  • run tests with systemProperty 'tests.rest.suite', 'old_cluster' + *
  • upgrade one node + *
  • run tests with systemProperty 'tests.rest.suite', 'mixed_cluster' + *
  • upgrade one more node + *
  • run tests with systemProperty 'tests.rest.suite', 'mixed_cluster' again + *
  • updgrade the last node + *
  • run tests with systemProperty 'tests.rest.suite', 'upgraded_cluster' + *
+ */ -test { - // CI doesn't like it when there's multiple clusters running at once - maxParallelForks = 1 -} + def baseCluster = testClusters.register(baseName) { + versions = [bwcVersion.toString(), project.version] + numberOfNodes = 3 -BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> - tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { - usesBwcDistribution(bwcVersion) - systemProperty("tests.old_cluster_version", bwcVersion) + setting 'repositories.url.allowed_urls', 'http://snapshot.test*' + setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'xpack.security.enabled', 'false' + setting 'logger.org.elasticsearch.cluster.service.MasterService', 'TRACE' + setting 'logger.org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator', 'TRACE' + setting 'logger.org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders', 'TRACE' + requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") + } + + String oldVersion = bwcVersion.toString() + + tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) { + dependsOn "processTestResources" + useCluster baseCluster + mustRunAfter("precommit") + doFirst { + delete("${buildDir}/cluster/shared/repo/${baseName}") + } + def excludeList = [] + systemProperty 'tests.rest.suite', 'old_cluster' + systemProperty 'tests.upgrade_from_version', oldVersion + nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) + nonInputProperties.systemProperty('tests.clustername', baseName) + if (bwcVersion.before("8.4.0")) { + excludeList.addAll(["old_cluster/30_vector_search/*"]) + } else if (bwcVersion.before("8.6.0")) { + excludeList.addAll(["old_cluster/30_vector_search/Create indexed byte vectors and search"]) + } + if (excludeList.isEmpty() == false) { + systemProperty 'tests.rest.blacklist', excludeList.join(',') + } + } + + tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) { + dependsOn "${baseName}#oldClusterTest" + useCluster baseCluster + doFirst { + baseCluster.get().nextNodeToNextVersion() + } + systemProperty 'tests.rest.suite', 'mixed_cluster' + systemProperty 'tests.upgrade_from_version', oldVersion + systemProperty 'tests.first_round', 'true' + nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) + nonInputProperties.systemProperty('tests.clustername', baseName) + def excludeList = [] + if (bwcVersion.before("8.4.0")) { + excludeList.addAll(["mixed_cluster/30_vector_search/*"]) + } else if (bwcVersion.before("8.6.0")) { + excludeList.addAll(["mixed_cluster/30_vector_search/Search byte indices created in old cluster"]) + } + if (excludeList.isEmpty() == false) { + systemProperty 'tests.rest.blacklist', excludeList.join(',') + } + } + + tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) { + dependsOn "${baseName}#oneThirdUpgradedTest" + useCluster baseCluster + doFirst { + baseCluster.get().nextNodeToNextVersion() + } + systemProperty 'tests.rest.suite', 'mixed_cluster' + systemProperty 'tests.upgrade_from_version', oldVersion + systemProperty 'tests.first_round', 'false' + nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) + nonInputProperties.systemProperty('tests.clustername', baseName) + def excludeList = [] + if (bwcVersion.before("8.4.0")) { + excludeList.addAll(["mixed_cluster/30_vector_search/*"]) + } else if (bwcVersion.before("8.6.0")) { + excludeList.addAll(["mixed_cluster/30_vector_search/Search byte indices created in old cluster"]) + } + if (excludeList.isEmpty() == false) { + systemProperty 'tests.rest.blacklist', excludeList.join(',') + } + } + + tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) { + dependsOn "${baseName}#twoThirdsUpgradedTest" + doFirst { + baseCluster.get().nextNodeToNextVersion() + } + useCluster testClusters.named(baseName) + systemProperty 'tests.rest.suite', 'upgraded_cluster' + systemProperty 'tests.upgrade_from_version', oldVersion + nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(","))) + nonInputProperties.systemProperty('tests.clustername', baseName) + def excludeList = [] + if (bwcVersion.before("8.4.0")) { + excludeList.addAll(["upgraded_cluster/30_vector_search/*"]) + } else if (bwcVersion.before("8.6.0")) { + excludeList.addAll(["upgraded_cluster/30_vector_search/Search byte indices created in old cluster"]) + } + if (excludeList.isEmpty() == false) { + systemProperty 'tests.rest.blacklist', excludeList.join(',') + } } -} -testClusters.configureEach { - setting 'xpack.security.enabled', 'false' + tasks.register(bwcTaskName(bwcVersion)) { + dependsOn tasks.named("${baseName}#upgradedClusterTest") + } } diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java deleted file mode 100644 index e1500358327c5..0000000000000 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.upgrades; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.FeatureFlag; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.elasticsearch.test.cluster.util.Version; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.test.rest.ObjectPath; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.rules.RuleChain; -import org.junit.rules.TemporaryFolder; -import org.junit.rules.TestRule; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.notNullValue; - -public abstract class ParameterizedRollingUpgradeTestCase extends ESRestTestCase { - private static final Version OLD_CLUSTER_VERSION = Version.fromString(System.getProperty("tests.old_cluster_version")); - - private static final TemporaryFolder repoDirectory = new TemporaryFolder(); - - private static final int NODE_NUM = 3; - - private static final ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .version(getOldClusterTestVersion()) - .nodes(NODE_NUM) - .setting("path.repo", new Supplier<>() { - @Override - @SuppressForbidden(reason = "TemporaryFolder only has io.File methods, not nio.File") - public String get() { - return repoDirectory.getRoot().getPath(); - } - }) - .setting("xpack.security.enabled", "false") - .feature(FeatureFlag.TIME_SERIES_MODE) - .build(); - - @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); - - @ParametersFactory(shuffle = false) - public static Iterable parameters() { - return Stream.concat(Stream.of((Integer) null), IntStream.range(0, NODE_NUM).boxed()).map(n -> new Object[] { n }).toList(); - } - - private static final Set upgradedNodes = new HashSet<>(); - private static boolean upgradeFailed = false; - private static IndexVersion oldIndexVersion; - - private final Integer requestedUpgradeNode; - - protected ParameterizedRollingUpgradeTestCase(@Name("upgradeNode") Integer upgradeNode) { - this.requestedUpgradeNode = upgradeNode; - } - - @Before - public void extractOldIndexVersion() throws Exception { - if (oldIndexVersion == null && upgradedNodes.isEmpty()) { - IndexVersion indexVersion = null; // these should all be the same version - - Request request = new Request("GET", "_nodes"); - request.addParameter("filter_path", "nodes.*.index_version,nodes.*.name"); - Response response = client().performRequest(request); - ObjectPath objectPath = ObjectPath.createFromResponse(response); - Map nodeMap = objectPath.evaluate("nodes"); - for (String id : nodeMap.keySet()) { - Number ix = objectPath.evaluate("nodes." + id + ".index_version"); - IndexVersion version; - if (ix != null) { - version = IndexVersion.fromId(ix.intValue()); - } else { - // it doesn't have index version (pre 8.11) - just infer it from the release version - version = IndexVersion.fromId(getOldClusterVersion().id); - } - - if (indexVersion == null) { - indexVersion = version; - } else { - String name = objectPath.evaluate("nodes." + id + ".name"); - assertThat("Node " + name + " has a different index version to other nodes", version, equalTo(indexVersion)); - } - } - - assertThat("Index version could not be read", indexVersion, notNullValue()); - oldIndexVersion = indexVersion; - } - } - - @Before - public void upgradeNode() throws Exception { - // Skip remaining tests if upgrade failed - assumeFalse("Cluster upgrade failed", upgradeFailed); - - if (requestedUpgradeNode != null && upgradedNodes.contains(requestedUpgradeNode) == false) { - closeClients(); - // we might be running a specific upgrade test by itself - check previous nodes too - for (int n = 0; n <= requestedUpgradeNode; n++) { - if (upgradedNodes.add(n)) { - try { - cluster.upgradeNodeToVersion(n, Version.CURRENT); - } catch (Exception e) { - upgradeFailed = true; - throw e; - } - } - } - initClient(); - } - } - - @AfterClass - public static void resetNodes() { - oldIndexVersion = null; - upgradedNodes.clear(); - upgradeFailed = false; - } - - protected static org.elasticsearch.Version getOldClusterVersion() { - return org.elasticsearch.Version.fromString(OLD_CLUSTER_VERSION.toString()); - } - - protected static IndexVersion getOldClusterIndexVersion() { - assert oldIndexVersion != null; - return oldIndexVersion; - } - - protected static Version getOldClusterTestVersion() { - return Version.fromString(OLD_CLUSTER_VERSION.toString()); - } - - protected static boolean isOldCluster() { - return upgradedNodes.isEmpty(); - } - - protected static boolean isFirstMixedCluster() { - return upgradedNodes.size() == 1; - } - - protected static boolean isMixedCluster() { - return upgradedNodes.isEmpty() == false && upgradedNodes.size() < NODE_NUM; - } - - protected static boolean isUpgradedCluster() { - return upgradedNodes.size() == NODE_NUM; - } - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } - - @Override - protected final boolean resetFeatureStates() { - return false; - } - - @Override - protected final boolean preserveIndicesUponCompletion() { - return true; - } - - @Override - protected final boolean preserveDataStreamsUponCompletion() { - return true; - } - - @Override - protected final boolean preserveReposUponCompletion() { - return true; - } - - @Override - protected boolean preserveTemplatesUponCompletion() { - return true; - } - - @Override - protected boolean preserveClusterUponCompletion() { - return true; - } - - @Override - protected final Settings restClientSettings() { - return Settings.builder() - .put(super.restClientSettings()) - // increase the timeout here to 90 seconds to handle long waits for a green - // cluster health. the waits for green need to be longer than a minute to - // account for delayed shards - .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") - .build(); - } - - @Override - protected final String getEnsureGreenTimeout() { - // increase the timeout here to 70 seconds to handle long waits for a green - // cluster health. the waits for green need to be longer than a minute to - // account for delayed shards - return "70s"; - } -} diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java deleted file mode 100644 index 3c6e15a78fc1f..0000000000000 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.upgrades; - -import com.carrotsearch.randomizedtesting.annotations.Name; - -import org.elasticsearch.Version; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.core.Strings; - -import java.io.IOException; -import java.util.Map; - -import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; -import static org.hamcrest.Matchers.is; - -public class UpgradeWithOldIndexSettingsIT extends ParameterizedRollingUpgradeTestCase { - - public UpgradeWithOldIndexSettingsIT(@Name("upgradeNode") Integer upgradeNode) { - super(upgradeNode); - } - - private static final String INDEX_NAME = "test_index_old_settings"; - private static final String EXPECTED_WARNING = "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will " - + "be removed in a future release! See the breaking changes documentation for the next major version."; - - private static final String EXPECTED_V8_WARNING = "[index.indexing.slowlog.level] setting was deprecated in the previous Elasticsearch" - + " release and is removed in this release."; - - public void testOldIndexSettings() throws Exception { - if (isOldCluster()) { - Request createTestIndex = new Request("PUT", "/" + INDEX_NAME); - createTestIndex.setJsonEntity("{\"settings\": {\"index.indexing.slowlog.level\": \"WARN\"}}"); - createTestIndex.setOptions(expectWarnings(EXPECTED_WARNING)); - if (getOldClusterVersion().before(Version.V_8_0_0)) { - // create index with settings no longer valid in 8.0 - client().performRequest(createTestIndex); - } else { - assertTrue( - expectThrows(ResponseException.class, () -> client().performRequest(createTestIndex)).getMessage() - .contains("unknown setting [index.indexing.slowlog.level]") - ); - - Request createTestIndex1 = new Request("PUT", "/" + INDEX_NAME); - client().performRequest(createTestIndex1); - } - - // add some data - Request bulk = new Request("POST", "/_bulk"); - bulk.addParameter("refresh", "true"); - if (getOldClusterVersion().before(Version.V_8_0_0)) { - bulk.setOptions(expectWarnings(EXPECTED_WARNING)); - } - bulk.setJsonEntity(Strings.format(""" - {"index": {"_index": "%s"}} - {"f1": "v1", "f2": "v2"} - """, INDEX_NAME)); - client().performRequest(bulk); - } else if (isMixedCluster()) { - // add some more data - Request bulk = new Request("POST", "/_bulk"); - bulk.addParameter("refresh", "true"); - if (getOldClusterVersion().before(Version.V_8_0_0)) { - bulk.setOptions(expectWarnings(EXPECTED_WARNING)); - } - bulk.setJsonEntity(Strings.format(""" - {"index": {"_index": "%s"}} - {"f1": "v3", "f2": "v4"} - """, INDEX_NAME)); - client().performRequest(bulk); - } else { - if (getOldClusterVersion().before(Version.V_8_0_0)) { - Request createTestIndex = new Request("PUT", "/" + INDEX_NAME + "/_settings"); - // update index settings should work - createTestIndex.setJsonEntity("{\"index.indexing.slowlog.level\": \"INFO\"}"); - createTestIndex.setOptions(expectWarnings(EXPECTED_V8_WARNING)); - client().performRequest(createTestIndex); - - // ensure we were able to change the setting, despite it having no effect - Request indexSettingsRequest = new Request("GET", "/" + INDEX_NAME + "/_settings"); - Map response = entityAsMap(client().performRequest(indexSettingsRequest)); - - var slowLogLevel = (String) (XContentMapValues.extractValue( - INDEX_NAME + ".settings.index.indexing.slowlog.level", - response - )); - - // check that we can read our old index settings - assertThat(slowLogLevel, is("INFO")); - } - assertCount(INDEX_NAME, 2); - } - } - - private void assertCount(String index, int countAtLeast) throws IOException { - Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search"); - searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); - searchTestIndexRequest.addParameter("filter_path", "hits.total"); - Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest); - Map response = entityAsMap(searchTestIndexResponse); - - var hitsTotal = (Integer) (XContentMapValues.extractValue("hits.total", response)); - - assertTrue(hitsTotal >= countAtLeast); - } - - public static void updateIndexSettingsPermittingSlowlogDeprecationWarning(String index, Settings.Builder settings) throws IOException { - Request request = new Request("PUT", "/" + index + "/_settings"); - request.setJsonEntity(org.elasticsearch.common.Strings.toString(settings.build())); - if (getOldClusterVersion().before(Version.V_7_17_9)) { - // There is a bug (fixed in 7.17.9 and 8.7.0 where deprecation warnings could leak into ClusterApplierService#applyChanges) - // Below warnings are set (and leaking) from an index in this test case - request.setOptions(expectVersionSpecificWarnings(v -> { - v.compatible( - "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will be removed in a future release! " - + "See the breaking changes documentation for the next major version." - ); - })); - } - client().performRequest(request); - } -} diff --git a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java similarity index 79% rename from qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java rename to qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java index 74a8eb7fd1988..008a718be5873 100644 --- a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java @@ -9,8 +9,11 @@ import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.rest.ESRestTestCase; +import static org.hamcrest.Matchers.lessThan; + public abstract class AbstractRollingTestCase extends ESRestTestCase { protected enum ClusterType { OLD, @@ -31,6 +34,16 @@ public static ClusterType parse(String value) { protected static final boolean FIRST_MIXED_ROUND = Boolean.parseBoolean(System.getProperty("tests.first_round", "false")); protected static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version")); + protected static IndexVersion getOldClusterIndexVersion() { + var version = UPGRADE_FROM_VERSION; + if (version.equals(org.elasticsearch.Version.CURRENT)) { + return IndexVersion.current(); + } else { + assertThat("Index version needs to be added to rolling test parameters", version, lessThan(org.elasticsearch.Version.V_8_11_0)); + return IndexVersion.fromId(version.id); + } + } + @Override protected final boolean resetFeatureStates() { return false; @@ -41,6 +54,11 @@ protected final boolean preserveIndicesUponCompletion() { return true; } + @Override + protected final boolean preserveDataStreamsUponCompletion() { + return true; + } + @Override protected final boolean preserveReposUponCompletion() { return true; diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java similarity index 82% rename from qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java rename to qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java index e945d457986d0..5bafccf7aee1b 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java @@ -8,8 +8,6 @@ package org.elasticsearch.upgrades; -import com.carrotsearch.randomizedtesting.annotations.Name; - import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesRequest; import org.elasticsearch.client.Request; @@ -27,33 +25,24 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; -public class DesiredNodesUpgradeIT extends ParameterizedRollingUpgradeTestCase { - - private final int desiredNodesVersion; - - public DesiredNodesUpgradeIT(@Name("upgradeNode") Integer upgradeNode) { - super(upgradeNode); - desiredNodesVersion = Objects.requireNonNullElse(upgradeNode, -1) + 2; - } - +public class DesiredNodesUpgradeIT extends AbstractRollingTestCase { private enum ProcessorsPrecision { DOUBLE, FLOAT } public void testUpgradeDesiredNodes() throws Exception { - assumeTrue("Desired nodes was introduced in 8.1", getOldClusterVersion().onOrAfter(Version.V_8_1_0)); + assumeTrue("Desired nodes was introduced in 8.1", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_1_0)); - if (getOldClusterVersion().onOrAfter(Processors.DOUBLE_PROCESSORS_SUPPORT_VERSION)) { + if (UPGRADE_FROM_VERSION.onOrAfter(Processors.DOUBLE_PROCESSORS_SUPPORT_VERSION)) { assertUpgradedNodesCanReadDesiredNodes(); - } else if (getOldClusterVersion().onOrAfter(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) { + } else if (UPGRADE_FROM_VERSION.onOrAfter(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) { assertDesiredNodesUpdatedWithRoundedUpFloatsAreIdempotent(); } else { assertDesiredNodesWithFloatProcessorsAreRejectedInOlderVersions(); @@ -61,7 +50,13 @@ public void testUpgradeDesiredNodes() throws Exception { } private void assertUpgradedNodesCanReadDesiredNodes() throws Exception { - if (isMixedCluster() || isUpgradedCluster()) { + final int desiredNodesVersion = switch (CLUSTER_TYPE) { + case OLD -> 1; + case MIXED -> FIRST_MIXED_ROUND ? 2 : 3; + case UPGRADED -> 4; + }; + + if (CLUSTER_TYPE != ClusterType.OLD) { final Map desiredNodes = getLatestDesiredNodes(); final String historyId = extractValue(desiredNodes, "history_id"); final int version = extractValue(desiredNodes, "version"); @@ -88,7 +83,13 @@ private void assertDesiredNodesUpdatedWithRoundedUpFloatsAreIdempotent() throws ) .toList(); - if (isMixedCluster() || isUpgradedCluster()) { + final int desiredNodesVersion = switch (CLUSTER_TYPE) { + case OLD -> 1; + case MIXED -> FIRST_MIXED_ROUND ? 2 : 3; + case UPGRADED -> 4; + }; + + if (CLUSTER_TYPE != ClusterType.OLD) { updateDesiredNodes(desiredNodes, desiredNodesVersion - 1); } for (int i = 0; i < 2; i++) { @@ -99,25 +100,28 @@ private void assertDesiredNodesUpdatedWithRoundedUpFloatsAreIdempotent() throws final int latestDesiredNodesVersion = extractValue(latestDesiredNodes, "version"); assertThat(latestDesiredNodesVersion, is(equalTo(desiredNodesVersion))); - if (isUpgradedCluster()) { + if (CLUSTER_TYPE == ClusterType.UPGRADED) { assertAllDesiredNodesAreActualized(); } } private void assertDesiredNodesWithFloatProcessorsAreRejectedInOlderVersions() throws Exception { - if (isOldCluster()) { - addClusterNodesToDesiredNodesWithIntegerProcessors(1); - } else if (isMixedCluster()) { - // Processor ranges or float processors are forbidden during upgrades: 8.2 -> 8.3 clusters - final var responseException = expectThrows( - ResponseException.class, - () -> addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(desiredNodesVersion, ProcessorsPrecision.FLOAT) - ); - final var statusCode = responseException.getResponse().getStatusLine().getStatusCode(); - assertThat(statusCode, is(equalTo(400))); - } else { - assertAllDesiredNodesAreActualized(); - addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(4, ProcessorsPrecision.FLOAT); + switch (CLUSTER_TYPE) { + case OLD -> addClusterNodesToDesiredNodesWithIntegerProcessors(1); + case MIXED -> { + int version = FIRST_MIXED_ROUND ? 2 : 3; + // Processor ranges or float processors are forbidden during upgrades: 8.2 -> 8.3 clusters + final var responseException = expectThrows( + ResponseException.class, + () -> addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(version, ProcessorsPrecision.FLOAT) + ); + final var statusCode = responseException.getResponse().getStatusLine().getStatusCode(); + assertThat(statusCode, is(equalTo(400))); + } + case UPGRADED -> { + assertAllDesiredNodesAreActualized(); + addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(4, ProcessorsPrecision.FLOAT); + } } getLatestDesiredNodes(); diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java similarity index 90% rename from qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java rename to qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java index 307e9946b7601..588802fb50709 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java @@ -8,8 +8,6 @@ package org.elasticsearch.upgrades; -import com.carrotsearch.randomizedtesting.annotations.Name; - import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; @@ -23,17 +21,14 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -public class FeatureUpgradeIT extends ParameterizedRollingUpgradeTestCase { - - public FeatureUpgradeIT(@Name("upgradeNode") Integer upgradeNode) { - super(upgradeNode); - } +public class FeatureUpgradeIT extends AbstractRollingTestCase { + @SuppressWarnings("unchecked") public void testGetFeatureUpgradeStatus() throws Exception { final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " + "access to system indices will be prevented by default"; - if (isOldCluster()) { + if (CLUSTER_TYPE == ClusterType.OLD) { // setup - put something in the tasks index // create index Request createTestIndex = new Request("PUT", "/feature_test_index_old"); @@ -84,7 +79,7 @@ public void testGetFeatureUpgradeStatus() throws Exception { } }); - } else if (isUpgradedCluster()) { + } else if (CLUSTER_TYPE == ClusterType.UPGRADED) { // check results assertBusy(() -> { Request clusterStateRequest = new Request("GET", "/_migration/system_features"); @@ -100,7 +95,7 @@ public void testGetFeatureUpgradeStatus() throws Exception { assertThat(feature, aMapWithSize(4)); assertThat(feature.get("minimum_index_version"), equalTo(getOldClusterIndexVersion().toString())); - if (getOldClusterVersion().before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION)) { + if (UPGRADE_FROM_VERSION.before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION)) { assertThat(feature.get("migration_status"), equalTo("MIGRATION_NEEDED")); } else { assertThat(feature.get("migration_status"), equalTo("NO_MIGRATION_NEEDED")); @@ -108,4 +103,5 @@ public void testGetFeatureUpgradeStatus() throws Exception { }); } } + } diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java similarity index 94% rename from qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java rename to qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java index 333cff3c4e039..83865222a8867 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java @@ -8,8 +8,6 @@ package org.elasticsearch.upgrades; -import com.carrotsearch.randomizedtesting.annotations.Name; - import org.apache.http.HttpHost; import org.elasticsearch.Version; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; @@ -38,17 +36,15 @@ * In 8.2 we also added the ability to filter fields by type and metadata, with some post-hoc filtering applied on * the co-ordinating node if older nodes were included in the system */ -public class FieldCapsIT extends ParameterizedRollingUpgradeTestCase { - - public FieldCapsIT(@Name("upgradeNode") Integer upgradeNode) { - super(upgradeNode); - } - - private static boolean oldIndicesCreated; - private static boolean newIndicesCreated; +public class FieldCapsIT extends AbstractRollingTestCase { + private static boolean indicesCreated = false; @Before public void setupIndices() throws Exception { + if (indicesCreated) { + return; + } + indicesCreated = true; final String redMapping = """ "properties": { "red_field": { "type": "keyword" }, @@ -67,7 +63,7 @@ public void setupIndices() throws Exception { "timestamp": {"type": "date"} } """; - if (isOldCluster() && oldIndicesCreated == false) { + if (CLUSTER_TYPE == ClusterType.OLD) { createIndex("old_red_1", Settings.EMPTY, redMapping); createIndex("old_red_2", Settings.EMPTY, redMapping); createIndex("old_red_empty", Settings.EMPTY, redMapping); @@ -82,8 +78,7 @@ public void setupIndices() throws Exception { ); assertOK(client().performRequest(indexRequest)); } - oldIndicesCreated = true; - } else if (isFirstMixedCluster() && newIndicesCreated == false) { + } else if (CLUSTER_TYPE == ClusterType.MIXED && FIRST_MIXED_ROUND) { createIndex("new_red_1", Settings.EMPTY, redMapping); createIndex("new_red_2", Settings.EMPTY, redMapping); createIndex("new_red_empty", Settings.EMPTY, redMapping); @@ -98,7 +93,6 @@ public void setupIndices() throws Exception { ); assertOK(client().performRequest(indexRequest)); } - newIndicesCreated = true; } } @@ -155,7 +149,7 @@ public void testOldIndicesWithIndexFilter() throws Exception { } public void testNewIndicesOnly() throws Exception { - assumeFalse("required mixed or upgraded cluster", isOldCluster()); + assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); { FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), null, null, null); assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2", "new_red_empty" })); @@ -183,7 +177,7 @@ public void testNewIndicesOnly() throws Exception { } public void testNewIndicesOnlyWithIndexFilter() throws Exception { - assumeFalse("required mixed or upgraded cluster", isOldCluster()); + assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12"); { FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), indexFilter, null, null); @@ -209,7 +203,7 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception { } public void testAllIndices() throws Exception { - assumeFalse("required mixed or upgraded cluster", isOldCluster()); + assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), null, null, null); assertThat( resp.getIndices(), @@ -241,7 +235,7 @@ public void testAllIndices() throws Exception { } public void testAllIndicesWithIndexFilter() throws Exception { - assumeFalse("required mixed or upgraded cluster", isOldCluster()); + assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12"); FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), indexFilter, null, null); assertThat( @@ -291,7 +285,7 @@ private RestClient getUpgradedNodeClient() throws IOException { // because we are testing that the upgraded node will correctly apply filtering // to responses from older nodes that don't understand the filter parameters public void testAllIndicesWithFieldTypeFilter() throws Exception { - assumeFalse("required mixed or upgraded cluster", isOldCluster()); + assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); RestClient restClient = getUpgradedNodeClient(); FieldCapabilitiesResponse resp = fieldCaps(restClient, List.of("old_*", "new_*"), List.of("*"), null, "keyword", null); assertThat(resp.getField("red_field").keySet(), contains("keyword")); @@ -304,7 +298,7 @@ public void testAllIndicesWithFieldTypeFilter() throws Exception { // because we are testing that the upgraded node will correctly apply filtering // to responses from older nodes that don't understand the filter parameters public void testAllIndicesWithExclusionFilter() throws Exception { - assumeFalse("required mixed or upgraded cluster", isOldCluster()); + assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD); RestClient client = getUpgradedNodeClient(); { FieldCapabilitiesResponse resp = fieldCaps(client, List.of("old_*", "new_*"), List.of("*"), null, null, null); diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java similarity index 52% rename from qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java rename to qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java index ecd327b0d66c8..b860e53d447b5 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java @@ -7,8 +7,6 @@ */ package org.elasticsearch.upgrades; -import com.carrotsearch.randomizedtesting.annotations.Name; - import org.apache.http.util.EntityUtils; import org.elasticsearch.Version; import org.elasticsearch.client.Request; @@ -17,6 +15,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.Booleans; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.test.ListMatcher; import org.elasticsearch.xcontent.XContentBuilder; @@ -41,36 +40,39 @@ /** * Basic test that indexed documents survive the rolling restart. See - * {@code RecoveryIT} for much more in depth testing of the mechanism + * {@link RecoveryIT} for much more in depth testing of the mechanism * by which they survive. *

* This test is an almost exact copy of IndexingIT in the * xpack rolling restart tests. We should work on a way to remove this * duplication but for now we have no real way to share code. */ -public class IndexingIT extends ParameterizedRollingUpgradeTestCase { - - public IndexingIT(@Name("upgradeNode") Integer upgradeNode) { - super(upgradeNode); - } +public class IndexingIT extends AbstractRollingTestCase { public void testIndexing() throws IOException { - if (isMixedCluster()) { - Request waitForYellow = new Request("GET", "/_cluster/health"); - waitForYellow.addParameter("wait_for_nodes", "3"); - waitForYellow.addParameter("wait_for_status", "yellow"); - client().performRequest(waitForYellow); - } else if (isUpgradedCluster()) { - Request waitForGreen = new Request("GET", "/_cluster/health/test_index,index_with_replicas,empty_index"); - waitForGreen.addParameter("wait_for_nodes", "3"); - waitForGreen.addParameter("wait_for_status", "green"); - // wait for long enough that we give delayed unassigned shards to stop being delayed - waitForGreen.addParameter("timeout", "70s"); - waitForGreen.addParameter("level", "shards"); - client().performRequest(waitForGreen); + switch (CLUSTER_TYPE) { + case OLD: + break; + case MIXED: + Request waitForYellow = new Request("GET", "/_cluster/health"); + waitForYellow.addParameter("wait_for_nodes", "3"); + waitForYellow.addParameter("wait_for_status", "yellow"); + client().performRequest(waitForYellow); + break; + case UPGRADED: + Request waitForGreen = new Request("GET", "/_cluster/health/test_index,index_with_replicas,empty_index"); + waitForGreen.addParameter("wait_for_nodes", "3"); + waitForGreen.addParameter("wait_for_status", "green"); + // wait for long enough that we give delayed unassigned shards to stop being delayed + waitForGreen.addParameter("timeout", "70s"); + waitForGreen.addParameter("level", "shards"); + client().performRequest(waitForGreen); + break; + default: + throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]"); } - if (isOldCluster()) { + if (CLUSTER_TYPE == ClusterType.OLD) { Request createTestIndex = new Request("PUT", "/test_index"); createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}"); useIgnoreMultipleMatchingTemplatesWarningsHandler(createTestIndex); @@ -93,20 +95,30 @@ public void testIndexing() throws IOException { } int expectedCount; - if (isOldCluster() || isFirstMixedCluster()) { - expectedCount = 5; - } else if (isMixedCluster()) { - expectedCount = 10; - } else { - expectedCount = 15; + switch (CLUSTER_TYPE) { + case OLD: + expectedCount = 5; + break; + case MIXED: + if (Booleans.parseBoolean(System.getProperty("tests.first_round"))) { + expectedCount = 5; + } else { + expectedCount = 10; + } + break; + case UPGRADED: + expectedCount = 15; + break; + default: + throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]"); } assertCount("test_index", expectedCount); assertCount("index_with_replicas", 5); assertCount("empty_index", 0); - if (isOldCluster() == false) { - bulk("test_index", "_" + (isMixedCluster() ? "MIXED" : "UPGRADED"), 5); + if (CLUSTER_TYPE != ClusterType.OLD) { + bulk("test_index", "_" + CLUSTER_TYPE, 5); Request toBeDeleted = new Request("PUT", "/test_index/_doc/to_be_deleted"); toBeDeleted.addParameter("refresh", "true"); toBeDeleted.setJsonEntity("{\"f1\": \"delete-me\"}"); @@ -131,76 +143,82 @@ public void testAutoIdWithOpTypeCreate() throws IOException { bulk.addParameter("refresh", "true"); bulk.setJsonEntity(b); - if (isOldCluster()) { - Request createTestIndex = new Request("PUT", "/" + indexName); - createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}"); - client().performRequest(createTestIndex); - } else if (isMixedCluster()) { - Request waitForGreen = new Request("GET", "/_cluster/health"); - waitForGreen.addParameter("wait_for_nodes", "3"); - client().performRequest(waitForGreen); - Version minNodeVersion = minNodeVersion(); - if (minNodeVersion.before(Version.V_7_5_0)) { - ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(bulk)); - assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); - assertThat( - e.getMessage(), - // if request goes to 7.5+ node - either(containsString("optype create not supported for indexing requests without explicit id until")) - // if request goes to < 7.5 node - .or(containsString("an id must be provided if version type or value are set")) - ); - } else { - client().performRequest(bulk); + switch (CLUSTER_TYPE) { + case OLD -> { + Request createTestIndex = new Request("PUT", "/" + indexName); + createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}"); + client().performRequest(createTestIndex); } - } else if (isUpgradedCluster()) { - client().performRequest(bulk); + case MIXED -> { + Request waitForGreen = new Request("GET", "/_cluster/health"); + waitForGreen.addParameter("wait_for_nodes", "3"); + client().performRequest(waitForGreen); + Version minNodeVersion = minNodeVersion(); + if (minNodeVersion.before(Version.V_7_5_0)) { + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(bulk)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + assertThat( + e.getMessage(), + // if request goes to 7.5+ node + either(containsString("optype create not supported for indexing requests without explicit id until")) + // if request goes to < 7.5 node + .or(containsString("an id must be provided if version type or value are set")) + ); + } else { + client().performRequest(bulk); + } + } + case UPGRADED -> client().performRequest(bulk); + default -> throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]"); } } public void testDateNanosFormatUpgrade() throws IOException { final String indexName = "test_date_nanos"; - if (isOldCluster()) { - Request createIndex = new Request("PUT", "/" + indexName); - XContentBuilder mappings = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .startObject("mappings") - .startObject("properties") - .startObject("date") - .field("type", "date") - .endObject() - .startObject("date_nanos") - .field("type", "date_nanos") - .endObject() - .endObject() - .endObject() - .endObject(); - createIndex.setJsonEntity(Strings.toString(mappings)); - client().performRequest(createIndex); - Request index = new Request("POST", "/" + indexName + "/_doc/"); - XContentBuilder doc = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .field("date", "2015-01-01T12:10:30.123456789Z") - .field("date_nanos", "2015-01-01T12:10:30.123456789Z") - .endObject(); - index.addParameter("refresh", "true"); - index.setJsonEntity(Strings.toString(doc)); - client().performRequest(index); - } else if (isUpgradedCluster()) { - Request search = new Request("POST", "/" + indexName + "/_search"); - XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .array("fields", new String[] { "date", "date_nanos" }) - .endObject(); - search.setJsonEntity(Strings.toString(query)); - Map response = entityAsMap(client().performRequest(search)); - Map bestHit = (Map) ((List) (XContentMapValues.extractValue("hits.hits", response))).get(0); - List date = (List) XContentMapValues.extractValue("fields.date", bestHit); - assertThat(date.size(), equalTo(1)); - assertThat(date.get(0), equalTo("2015-01-01T12:10:30.123Z")); - List dateNanos = (List) XContentMapValues.extractValue("fields.date_nanos", bestHit); - assertThat(dateNanos.size(), equalTo(1)); - assertThat(dateNanos.get(0), equalTo("2015-01-01T12:10:30.123456789Z")); + switch (CLUSTER_TYPE) { + case OLD -> { + Request createIndex = new Request("PUT", "/" + indexName); + XContentBuilder mappings = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .startObject("mappings") + .startObject("properties") + .startObject("date") + .field("type", "date") + .endObject() + .startObject("date_nanos") + .field("type", "date_nanos") + .endObject() + .endObject() + .endObject() + .endObject(); + createIndex.setJsonEntity(Strings.toString(mappings)); + client().performRequest(createIndex); + Request index = new Request("POST", "/" + indexName + "/_doc/"); + XContentBuilder doc = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .field("date", "2015-01-01T12:10:30.123456789Z") + .field("date_nanos", "2015-01-01T12:10:30.123456789Z") + .endObject(); + index.addParameter("refresh", "true"); + index.setJsonEntity(Strings.toString(doc)); + client().performRequest(index); + } + case UPGRADED -> { + Request search = new Request("POST", "/" + indexName + "/_search"); + XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent()) + .startObject() + .array("fields", new String[] { "date", "date_nanos" }) + .endObject(); + search.setJsonEntity(Strings.toString(query)); + Map response = entityAsMap(client().performRequest(search)); + Map bestHit = (Map) ((List) (XContentMapValues.extractValue("hits.hits", response))).get(0); + List date = (List) XContentMapValues.extractValue("fields.date", bestHit); + assertThat(date.size(), equalTo(1)); + assertThat(date.get(0), equalTo("2015-01-01T12:10:30.123Z")); + List dateNanos = (List) XContentMapValues.extractValue("fields.date_nanos", bestHit); + assertThat(dateNanos.size(), equalTo(1)); + assertThat(dateNanos.get(0), equalTo("2015-01-01T12:10:30.123456789Z")); + } } } @@ -229,45 +247,51 @@ private void bulk(String index, String valueSuffix, int count) throws IOExceptio } public void testTsdb() throws IOException { - assumeTrue("indexing time series indices changed in 8.2.0", getOldClusterVersion().onOrAfter(Version.V_8_2_0)); + assumeTrue("indexing time series indices changed in 8.2.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_2_0)); StringBuilder bulk = new StringBuilder(); - if (isOldCluster()) { - createTsdbIndex(); - tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[0], TSDB_TIMES[1], 0.1); - tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[0], TSDB_TIMES[1], -0.1); - bulk("tsdb", bulk.toString()); - assertTsdbAgg(closeTo(215.95, 0.005), closeTo(-215.95, 0.005)); - return; - } else if (isFirstMixedCluster()) { - tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[1], TSDB_TIMES[2], 0.1); - tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[1], TSDB_TIMES[2], -0.1); - tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[0], TSDB_TIMES[2], 1.1); - bulk("tsdb", bulk.toString()); - assertTsdbAgg(closeTo(217.45, 0.005), closeTo(-217.45, 0.005), closeTo(2391.95, 0.005)); - - } else if (isMixedCluster()) { - tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[2], TSDB_TIMES[3], 0.1); - tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[2], TSDB_TIMES[3], -0.1); - tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[2], TSDB_TIMES[3], 1.1); - tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[0], TSDB_TIMES[3], 10); - bulk("tsdb", bulk.toString()); - assertTsdbAgg(closeTo(218.95, 0.005), closeTo(-218.95, 0.005), closeTo(2408.45, 0.005), closeTo(21895, 0.5)); - return; - } else { - tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[3], TSDB_TIMES[4], 0.1); - tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[3], TSDB_TIMES[4], -0.1); - tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[3], TSDB_TIMES[4], 1.1); - tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[3], TSDB_TIMES[4], 10); - tsdbBulk(bulk, TSDB_DIMS.get(4), TSDB_TIMES[0], TSDB_TIMES[4], -5); - bulk("tsdb", bulk.toString()); - assertTsdbAgg( - closeTo(220.45, 0.005), - closeTo(-220.45, 0.005), - closeTo(2424.95, 0.005), - closeTo(22045, 0.5), - closeTo(-11022.5, 0.5) - ); + switch (CLUSTER_TYPE) { + case OLD -> { + createTsdbIndex(); + tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[0], TSDB_TIMES[1], 0.1); + tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[0], TSDB_TIMES[1], -0.1); + bulk("tsdb", bulk.toString()); + assertTsdbAgg(closeTo(215.95, 0.005), closeTo(-215.95, 0.005)); + return; + } + case MIXED -> { + if (FIRST_MIXED_ROUND) { + tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[1], TSDB_TIMES[2], 0.1); + tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[1], TSDB_TIMES[2], -0.1); + tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[0], TSDB_TIMES[2], 1.1); + bulk("tsdb", bulk.toString()); + assertTsdbAgg(closeTo(217.45, 0.005), closeTo(-217.45, 0.005), closeTo(2391.95, 0.005)); + return; + } + tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[2], TSDB_TIMES[3], 0.1); + tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[2], TSDB_TIMES[3], -0.1); + tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[2], TSDB_TIMES[3], 1.1); + tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[0], TSDB_TIMES[3], 10); + bulk("tsdb", bulk.toString()); + assertTsdbAgg(closeTo(218.95, 0.005), closeTo(-218.95, 0.005), closeTo(2408.45, 0.005), closeTo(21895, 0.5)); + return; + } + case UPGRADED -> { + tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[3], TSDB_TIMES[4], 0.1); + tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[3], TSDB_TIMES[4], -0.1); + tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[3], TSDB_TIMES[4], 1.1); + tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[3], TSDB_TIMES[4], 10); + tsdbBulk(bulk, TSDB_DIMS.get(4), TSDB_TIMES[0], TSDB_TIMES[4], -5); + bulk("tsdb", bulk.toString()); + assertTsdbAgg( + closeTo(220.45, 0.005), + closeTo(-220.45, 0.005), + closeTo(2424.95, 0.005), + closeTo(22045, 0.5), + closeTo(-11022.5, 0.5) + ); + return; + } } } @@ -337,60 +361,67 @@ private void assertTsdbAgg(Matcher... expected) throws IOException { } public void testSyntheticSource() throws IOException { - assumeTrue("added in 8.4.0", getOldClusterVersion().onOrAfter(Version.V_8_4_0)); - - if (isOldCluster()) { - Request createIndex = new Request("PUT", "/synthetic"); - XContentBuilder indexSpec = XContentBuilder.builder(XContentType.JSON.xContent()).startObject(); - indexSpec.startObject("mappings"); - { - indexSpec.startObject("_source").field("mode", "synthetic").endObject(); - indexSpec.startObject("properties").startObject("kwd").field("type", "keyword").endObject().endObject(); + assumeTrue("added in 8.4.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_4_0)); + + switch (CLUSTER_TYPE) { + case OLD -> { + Request createIndex = new Request("PUT", "/synthetic"); + XContentBuilder indexSpec = XContentBuilder.builder(XContentType.JSON.xContent()).startObject(); + indexSpec.startObject("mappings"); + { + indexSpec.startObject("_source").field("mode", "synthetic").endObject(); + indexSpec.startObject("properties").startObject("kwd").field("type", "keyword").endObject().endObject(); + } + indexSpec.endObject(); + createIndex.setJsonEntity(Strings.toString(indexSpec.endObject())); + client().performRequest(createIndex); + bulk("synthetic", """ + {"index": {"_index": "synthetic", "_id": "old"}} + {"kwd": "old", "int": -12} + """); + break; + } + case MIXED -> { + if (FIRST_MIXED_ROUND) { + bulk("synthetic", """ + {"index": {"_index": "synthetic", "_id": "mixed_1"}} + {"kwd": "mixed_1", "int": 22} + """); + } else { + bulk("synthetic", """ + {"index": {"_index": "synthetic", "_id": "mixed_2"}} + {"kwd": "mixed_2", "int": 33} + """); + } + break; + } + case UPGRADED -> { + bulk("synthetic", """ + {"index": {"_index": "synthetic", "_id": "new"}} + {"kwd": "new", "int": 21341325} + """); } - indexSpec.endObject(); - createIndex.setJsonEntity(Strings.toString(indexSpec.endObject())); - client().performRequest(createIndex); - bulk("synthetic", """ - {"index": {"_index": "synthetic", "_id": "old"}} - {"kwd": "old", "int": -12} - """); - } else if (isFirstMixedCluster()) { - bulk("synthetic", """ - {"index": {"_index": "synthetic", "_id": "mixed_1"}} - {"kwd": "mixed_1", "int": 22} - """); - } else if (isMixedCluster()) { - bulk("synthetic", """ - {"index": {"_index": "synthetic", "_id": "mixed_2"}} - {"kwd": "mixed_2", "int": 33} - """); - - } else { - bulk("synthetic", """ - {"index": {"_index": "synthetic", "_id": "new"}} - {"kwd": "new", "int": 21341325} - """); } assertMap( entityAsMap(client().performRequest(new Request("GET", "/synthetic/_doc/old"))), matchesMap().extraOk().entry("_source", matchesMap().entry("kwd", "old").entry("int", -12)) ); - if (isOldCluster()) { + if (CLUSTER_TYPE == ClusterType.OLD) { return; } assertMap( entityAsMap(client().performRequest(new Request("GET", "/synthetic/_doc/mixed_1"))), matchesMap().extraOk().entry("_source", matchesMap().entry("kwd", "mixed_1").entry("int", 22)) ); - if (isFirstMixedCluster()) { + if (CLUSTER_TYPE == ClusterType.MIXED && FIRST_MIXED_ROUND) { return; } assertMap( entityAsMap(client().performRequest(new Request("GET", "/synthetic/_doc/mixed_2"))), matchesMap().extraOk().entry("_source", matchesMap().entry("kwd", "mixed_2").entry("int", 33)) ); - if (isMixedCluster()) { + if (CLUSTER_TYPE == ClusterType.MIXED) { return; } assertMap( diff --git a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java similarity index 97% rename from qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java rename to qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index 077eae88fba02..39700514cd79f 100644 --- a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -41,6 +41,7 @@ import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING; import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING; import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY; +import static org.elasticsearch.upgrades.UpgradeWithOldIndexSettingsIT.updateIndexSettingsPermittingSlowlogDeprecationWarning; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.in; @@ -746,23 +747,4 @@ public void testSoftDeletesDisabledWarning() throws Exception { ensureGreen(indexName); indexDocs(indexName, randomInt(100), randomInt(100)); } - - /* - * Copied from UpgradeWithOldIndexSettingsIT in the new format - */ - private static void updateIndexSettingsPermittingSlowlogDeprecationWarning(String index, Settings.Builder settings) throws IOException { - Request request = new Request("PUT", "/" + index + "/_settings"); - request.setJsonEntity(org.elasticsearch.common.Strings.toString(settings.build())); - if (UPGRADE_FROM_VERSION.before(Version.V_7_17_9)) { - // There is a bug (fixed in 7.17.9 and 8.7.0 where deprecation warnings could leak into ClusterApplierService#applyChanges) - // Below warnings are set (and leaking) from an index in this test case - request.setOptions(expectVersionSpecificWarnings(v -> { - v.compatible( - "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will be removed in a future release! " - + "See the breaking changes documentation for the next major version." - ); - })); - } - client().performRequest(request); - } } diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java similarity index 61% rename from qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java rename to qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java index a01d379f68e76..b8ce7fe7a6fbb 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java @@ -8,8 +8,6 @@ package org.elasticsearch.upgrades; -import com.carrotsearch.randomizedtesting.annotations.Name; - import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.util.EntityUtils; @@ -42,102 +40,100 @@ import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.notNullValue; -public class SnapshotBasedRecoveryIT extends ParameterizedRollingUpgradeTestCase { - - public SnapshotBasedRecoveryIT(@Name("upgradeNode") Integer upgradeNode) { - super(upgradeNode); - } +public class SnapshotBasedRecoveryIT extends AbstractRollingTestCase { public void testSnapshotBasedRecovery() throws Exception { assumeFalse( "Cancel shard allocation command is broken for initial desired balance versions and might allocate shard " + "on the node where it is not supposed to be. Fixed by https://github.com/elastic/elasticsearch/pull/93635", - getOldClusterVersion() == Version.V_8_6_0 - || getOldClusterVersion() == Version.V_8_6_1 - || getOldClusterVersion() == Version.V_8_7_0 + UPGRADE_FROM_VERSION == Version.V_8_6_0 || UPGRADE_FROM_VERSION == Version.V_8_6_1 || UPGRADE_FROM_VERSION == Version.V_8_7_0 ); final String indexName = "snapshot_based_recovery"; final String repositoryName = "snapshot_based_recovery_repo"; final int numDocs = 200; - if (isOldCluster()) { - Settings.Builder settings = Settings.builder() - .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) - .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") - .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster - createIndex(indexName, settings.build()); - ensureGreen(indexName); - indexDocs(indexName, numDocs); - flush(indexName, true); - registerRepository( - repositoryName, - "fs", - true, - Settings.builder() - .put("location", "./snapshot_based_recovery") - .put(BlobStoreRepository.USE_FOR_PEER_RECOVERY_SETTING.getKey(), true) - .build() - ); - createSnapshot(repositoryName, "snap", true); - updateIndexSettings(indexName, Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)); - ensureGreen(indexName); - } else { - if (isFirstMixedCluster()) { - List upgradedNodeIds = getUpgradedNodeIds(); - // It's possible that the test simply does a rolling-restart, i.e. it "upgrades" to - // the same version. In that case we proceed without excluding any node - if (upgradedNodeIds.isEmpty() == false) { - assertThat(upgradedNodeIds.size(), is(equalTo(1))); - String upgradedNodeId = upgradedNodeIds.get(0); - logger.info("--> excluding [{}] from node [{}]", indexName, upgradedNodeId); - updateIndexSettings(indexName, Settings.builder().put("index.routing.allocation.exclude._id", upgradedNodeId)); - ensureGreen(indexName); - logger.info("--> finished excluding [{}] from node [{}]", indexName, upgradedNodeId); - } else { - logger.info("--> no upgrading nodes, not adding any exclusions for [{}]", indexName); - } + switch (CLUSTER_TYPE) { + case OLD -> { + Settings.Builder settings = Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) + .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") + .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster + createIndex(indexName, settings.build()); + ensureGreen(indexName); + indexDocs(indexName, numDocs); + flush(indexName, true); + registerRepository( + repositoryName, + "fs", + true, + Settings.builder() + .put("location", "./snapshot_based_recovery") + .put(BlobStoreRepository.USE_FOR_PEER_RECOVERY_SETTING.getKey(), true) + .build() + ); + createSnapshot(repositoryName, "snap", true); + updateIndexSettings(indexName, Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)); + ensureGreen(indexName); + } + case MIXED, UPGRADED -> { + if (FIRST_MIXED_ROUND) { + List upgradedNodeIds = getUpgradedNodeIds(); + // It's possible that the test simply does a rolling-restart, i.e. it "upgrades" to + // the same version. In that case we proceed without excluding any node + if (upgradedNodeIds.isEmpty() == false) { + assertThat(upgradedNodeIds.size(), is(equalTo(1))); + String upgradedNodeId = upgradedNodeIds.get(0); + logger.info("--> excluding [{}] from node [{}]", indexName, upgradedNodeId); + updateIndexSettings(indexName, Settings.builder().put("index.routing.allocation.exclude._id", upgradedNodeId)); + ensureGreen(indexName); + logger.info("--> finished excluding [{}] from node [{}]", indexName, upgradedNodeId); + } else { + logger.info("--> no upgrading nodes, not adding any exclusions for [{}]", indexName); + } - String primaryNodeId = getPrimaryNodeIdOfShard(indexName, 0); - Version primaryNodeVersion = getNodeVersion(primaryNodeId); + String primaryNodeId = getPrimaryNodeIdOfShard(indexName, 0); + Version primaryNodeVersion = getNodeVersion(primaryNodeId); - // Sometimes the primary shard ends on the upgraded node (i.e. after a rebalance) - // This causes issues when removing and adding replicas, since then we cannot allocate to any of the old nodes. - // That is an issue only for the first mixed round. - // In that case we exclude the upgraded node from the shard allocation and cancel the shard to force moving - // the primary to a node in the old version, this allows adding replicas in the first mixed round. - logger.info("--> Primary node in first mixed round {} / {}", primaryNodeId, primaryNodeVersion); - if (primaryNodeVersion.after(getOldClusterVersion())) { - logger.info("--> cancelling primary shard on node [{}]", primaryNodeId); - cancelShard(indexName, 0, primaryNodeId); - logger.info("--> done cancelling primary shard on node [{}]", primaryNodeId); + // Sometimes the primary shard ends on the upgraded node (i.e. after a rebalance) + // This causes issues when removing and adding replicas, since then we cannot allocate to any of the old nodes. + // That is an issue only for the first mixed round. + // In that case we exclude the upgraded node from the shard allocation and cancel the shard to force moving + // the primary to a node in the old version, this allows adding replicas in the first mixed round. + logger.info("--> Primary node in first mixed round {} / {}", primaryNodeId, primaryNodeVersion); + if (primaryNodeVersion.after(UPGRADE_FROM_VERSION)) { + logger.info("--> cancelling primary shard on node [{}]", primaryNodeId); + cancelShard(indexName, 0, primaryNodeId); + logger.info("--> done cancelling primary shard on node [{}]", primaryNodeId); - String currentPrimaryNodeId = getPrimaryNodeIdOfShard(indexName, 0); - assertThat(getNodeVersion(currentPrimaryNodeId), is(equalTo(getOldClusterVersion()))); + String currentPrimaryNodeId = getPrimaryNodeIdOfShard(indexName, 0); + assertThat(getNodeVersion(currentPrimaryNodeId), is(equalTo(UPGRADE_FROM_VERSION))); + } + } else { + logger.info("--> not in first upgrade round, removing exclusions for [{}]", indexName); + updateIndexSettings(indexName, Settings.builder().putNull("index.routing.allocation.exclude._id")); + logger.info("--> done removing exclusions for [{}]", indexName); } - } else { - logger.info("--> not in first upgrade round, removing exclusions for [{}]", indexName); - updateIndexSettings(indexName, Settings.builder().putNull("index.routing.allocation.exclude._id")); - logger.info("--> done removing exclusions for [{}]", indexName); - } - // Drop replicas - logger.info("--> dropping replicas from [{}]", indexName); - updateIndexSettingsPermittingSlowlogDeprecationWarning( - indexName, - Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) - ); - logger.info("--> finished dropping replicas from [{}], adding them back", indexName); - updateIndexSettingsPermittingSlowlogDeprecationWarning( - indexName, - Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1) - ); - logger.info("--> finished adding replicas from [{}]", indexName); - ensureGreen(indexName); + // Drop replicas + logger.info("--> dropping replicas from [{}]", indexName); + updateIndexSettingsPermittingSlowlogDeprecationWarning( + indexName, + Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) + ); + logger.info("--> finished dropping replicas from [{}], adding them back", indexName); + updateIndexSettingsPermittingSlowlogDeprecationWarning( + indexName, + Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1) + ); + logger.info("--> finished adding replicas from [{}]", indexName); + ensureGreen(indexName); - assertMatchAllReturnsAllDocuments(indexName, numDocs); - assertMatchQueryReturnsAllDocuments(indexName, numDocs); + assertMatchAllReturnsAllDocuments(indexName, numDocs); + assertMatchQueryReturnsAllDocuments(indexName, numDocs); + } + default -> throw new IllegalStateException("unknown type " + CLUSTER_TYPE); } } @@ -149,7 +145,7 @@ private List getUpgradedNodeIds() throws IOException { List upgradedNodes = new ArrayList<>(); for (Map.Entry> nodeInfoEntry : nodes.entrySet()) { Version nodeVersion = Version.fromString(extractValue(nodeInfoEntry.getValue(), "version")); - if (nodeVersion.after(getOldClusterVersion())) { + if (nodeVersion.after(UPGRADE_FROM_VERSION)) { upgradedNodes.add(nodeInfoEntry.getKey()); } } diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java similarity index 95% rename from qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java rename to qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java index f6de3f4e48d68..48c3007bb2674 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java @@ -8,8 +8,6 @@ package org.elasticsearch.upgrades; -import com.carrotsearch.randomizedtesting.annotations.Name; - import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.index.IndexVersion; @@ -23,17 +21,13 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -public class SystemIndicesUpgradeIT extends ParameterizedRollingUpgradeTestCase { - - public SystemIndicesUpgradeIT(@Name("upgradeNode") Integer upgradeNode) { - super(upgradeNode); - } +public class SystemIndicesUpgradeIT extends AbstractRollingTestCase { @SuppressWarnings("unchecked") public void testSystemIndicesUpgrades() throws Exception { final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " + "access to system indices will be prevented by default"; - if (isOldCluster()) { + if (CLUSTER_TYPE == ClusterType.OLD) { // create index Request createTestIndex = new Request("PUT", "/test_index_old"); createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}"); @@ -105,7 +99,7 @@ public void testSystemIndicesUpgrades() throws Exception { })); assertThat(client().performRequest(putAliasRequest).getStatusLine().getStatusCode(), is(200)); } - } else if (isUpgradedCluster()) { + } else if (CLUSTER_TYPE == ClusterType.UPGRADED) { assertBusy(() -> { Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata"); Map indices = new JsonMapView(entityAsMap(client().performRequest(clusterStateRequest))).get( diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TsdbIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TsdbIT.java similarity index 90% rename from qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TsdbIT.java rename to qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TsdbIT.java index f8464be894ac9..19f24c97a47f8 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TsdbIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TsdbIT.java @@ -8,8 +8,6 @@ package org.elasticsearch.upgrades; -import com.carrotsearch.randomizedtesting.annotations.Name; - import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.common.time.DateFormatter; @@ -26,11 +24,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class TsdbIT extends ParameterizedRollingUpgradeTestCase { - - public TsdbIT(@Name("upgradeNode") Integer upgradeNode) { - super(upgradeNode); - } +public class TsdbIT extends AbstractRollingTestCase { private static final String TEMPLATE = """ { @@ -94,21 +88,21 @@ public TsdbIT(@Name("upgradeNode") Integer upgradeNode) { private static final String BULK = """ {"create": {}} - {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}} + {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507","ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}} {"create": {}} - {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "hamster", "uid":"947e4ced-1786-4e53-9e0c-5c447e959508", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}} + {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "hamster", "uid":"947e4ced-1786-4e53-9e0c-5c447e959508","ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}} {"create": {}} - {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cow", "uid":"947e4ced-1786-4e53-9e0c-5c447e959509", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}} + {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cow", "uid":"947e4ced-1786-4e53-9e0c-5c447e959509","ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}} {"create": {}} - {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "rat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959510", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}} + {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "rat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959510","ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}} {"create": {}} - {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}} + {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9","ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}} {"create": {}} - {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "tiger", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea10", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}} + {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "tiger", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea10","ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}} {"create": {}} - {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "lion", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876e11", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}} + {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "lion", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876e11","ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}} {"create": {}} - {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "elephant", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876eb4", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}} + {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "elephant", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876eb4","ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}} """; private static final String DOC = """ @@ -131,11 +125,11 @@ public TsdbIT(@Name("upgradeNode") Integer upgradeNode) { public void testTsdbDataStream() throws Exception { assumeTrue( - "Skipping version [" + getOldClusterVersion() + "], because TSDB was GA-ed in 8.7.0", - getOldClusterVersion().onOrAfter(Version.V_8_7_0) + "Skipping version [" + UPGRADE_FROM_VERSION + "], because TSDB was GA-ed in 8.7.0", + UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_7_0) ); String dataStreamName = "k8s"; - if (isOldCluster()) { + if (CLUSTER_TYPE == ClusterType.OLD) { final String INDEX_TEMPLATE = """ { "index_patterns": ["$PATTERN"], @@ -150,20 +144,20 @@ public void testTsdbDataStream() throws Exception { assertOK(client().performRequest(putIndexTemplateRequest)); performOldClustertOperations(templateName, dataStreamName); - } else if (isMixedCluster()) { + } else if (CLUSTER_TYPE == ClusterType.MIXED) { performMixedClusterOperations(dataStreamName); - } else if (isUpgradedCluster()) { + } else if (CLUSTER_TYPE == ClusterType.UPGRADED) { performUpgradedClusterOperations(dataStreamName); } } public void testTsdbDataStreamWithComponentTemplate() throws Exception { assumeTrue( - "Skipping version [" + getOldClusterVersion() + "], because TSDB was GA-ed in 8.7.0 and bug was fixed in 8.11.0", - getOldClusterVersion().onOrAfter(Version.V_8_7_0) && getOldClusterVersion().before(Version.V_8_11_0) + "Skipping version [" + UPGRADE_FROM_VERSION + "], because TSDB was GA-ed in 8.7.0 and bug was fixed in 8.11.0", + UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_7_0) && UPGRADE_FROM_VERSION.before(Version.V_8_11_0) ); String dataStreamName = "template-with-component-template"; - if (isOldCluster()) { + if (CLUSTER_TYPE == ClusterType.OLD) { final String COMPONENT_TEMPLATE = """ { "template": $TEMPLATE @@ -187,9 +181,9 @@ public void testTsdbDataStreamWithComponentTemplate() throws Exception { assertOK(client().performRequest(putIndexTemplateRequest)); performOldClustertOperations(templateName, dataStreamName); - } else if (isMixedCluster()) { + } else if (CLUSTER_TYPE == ClusterType.MIXED) { performMixedClusterOperations(dataStreamName); - } else if (isUpgradedCluster()) { + } else if (CLUSTER_TYPE == ClusterType.UPGRADED) { performUpgradedClusterOperations(dataStreamName); var dataStreams = getDataStream(dataStreamName); @@ -248,7 +242,7 @@ private void performUpgradedClusterOperations(String dataStreamName) throws Exce private static void performMixedClusterOperations(String dataStreamName) throws IOException { ensureHealth(dataStreamName, request -> request.addParameter("wait_for_status", "yellow")); - if (isFirstMixedCluster()) { + if (FIRST_MIXED_ROUND) { indexDoc(dataStreamName); } assertSearch(dataStreamName, 9); diff --git a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java similarity index 95% rename from qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java rename to qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index 068747d5a4824..0f829f20fe3c4 100644 --- a/qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -40,6 +40,11 @@ protected boolean preserveTemplatesUponCompletion() { return true; } + @Override + protected boolean preserveDataStreamsUponCompletion() { + return true; + } + public UpgradeClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) { super(testCandidate); } diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java new file mode 100644 index 0000000000000..35688e7c244cf --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.upgrades; + +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.Strings; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; +import static org.hamcrest.Matchers.is; + +public class UpgradeWithOldIndexSettingsIT extends AbstractRollingTestCase { + + private static final String INDEX_NAME = "test_index_old_settings"; + private static final String EXPECTED_WARNING = "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will " + + "be removed in a future release! See the breaking changes documentation for the next major version."; + + private static final String EXPECTED_V8_WARNING = "[index.indexing.slowlog.level] setting was deprecated in the previous Elasticsearch" + + " release and is removed in this release."; + + @SuppressWarnings("unchecked") + public void testOldIndexSettings() throws Exception { + switch (CLUSTER_TYPE) { + case OLD -> { + Request createTestIndex = new Request("PUT", "/" + INDEX_NAME); + createTestIndex.setJsonEntity("{\"settings\": {\"index.indexing.slowlog.level\": \"WARN\"}}"); + createTestIndex.setOptions(expectWarnings(EXPECTED_WARNING)); + if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) { + // create index with settings no longer valid in 8.0 + client().performRequest(createTestIndex); + } else { + assertTrue( + expectThrows(ResponseException.class, () -> client().performRequest(createTestIndex)).getMessage() + .contains("unknown setting [index.indexing.slowlog.level]") + ); + + Request createTestIndex1 = new Request("PUT", "/" + INDEX_NAME); + client().performRequest(createTestIndex1); + } + + // add some data + Request bulk = new Request("POST", "/_bulk"); + bulk.addParameter("refresh", "true"); + if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) { + bulk.setOptions(expectWarnings(EXPECTED_WARNING)); + } + bulk.setJsonEntity(Strings.format(""" + {"index": {"_index": "%s"}} + {"f1": "v1", "f2": "v2"} + """, INDEX_NAME)); + client().performRequest(bulk); + } + case MIXED -> { + // add some more data + Request bulk = new Request("POST", "/_bulk"); + bulk.addParameter("refresh", "true"); + if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) { + bulk.setOptions(expectWarnings(EXPECTED_WARNING)); + } + bulk.setJsonEntity(Strings.format(""" + {"index": {"_index": "%s"}} + {"f1": "v3", "f2": "v4"} + """, INDEX_NAME)); + client().performRequest(bulk); + } + case UPGRADED -> { + if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) { + Request createTestIndex = new Request("PUT", "/" + INDEX_NAME + "/_settings"); + // update index settings should work + createTestIndex.setJsonEntity("{\"index.indexing.slowlog.level\": \"INFO\"}"); + createTestIndex.setOptions(expectWarnings(EXPECTED_V8_WARNING)); + client().performRequest(createTestIndex); + + // ensure we were able to change the setting, despite it having no effect + Request indexSettingsRequest = new Request("GET", "/" + INDEX_NAME + "/_settings"); + Map response = entityAsMap(client().performRequest(indexSettingsRequest)); + + var slowLogLevel = (String) (XContentMapValues.extractValue( + INDEX_NAME + ".settings.index.indexing.slowlog.level", + response + )); + + // check that we can read our old index settings + assertThat(slowLogLevel, is("INFO")); + } + assertCount(INDEX_NAME, 2); + } + } + } + + private void assertCount(String index, int countAtLeast) throws IOException { + Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search"); + searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); + searchTestIndexRequest.addParameter("filter_path", "hits.total"); + Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest); + Map response = entityAsMap(searchTestIndexResponse); + + var hitsTotal = (Integer) (XContentMapValues.extractValue("hits.total", response)); + + assertTrue(hitsTotal >= countAtLeast); + } + + public static void updateIndexSettingsPermittingSlowlogDeprecationWarning(String index, Settings.Builder settings) throws IOException { + Request request = new Request("PUT", "/" + index + "/_settings"); + request.setJsonEntity(org.elasticsearch.common.Strings.toString(settings.build())); + if (UPGRADE_FROM_VERSION.before(Version.V_7_17_9)) { + // There is a bug (fixed in 7.17.9 and 8.7.0 where deprecation warnings could leak into ClusterApplierService#applyChanges) + // Below warnings are set (and leaking) from an index in this test case + request.setOptions(expectVersionSpecificWarnings(v -> { + v.compatible( + "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will be removed in a future release! " + + "See the breaking changes documentation for the next major version." + ); + })); + } + client().performRequest(request); + } +} diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/XPackIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java similarity index 93% rename from qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/XPackIT.java rename to qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java index b63e1147442a7..40e63b4ae32d7 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/XPackIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java @@ -7,8 +7,6 @@ */ package org.elasticsearch.upgrades; -import com.carrotsearch.randomizedtesting.annotations.Name; - import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; import org.junit.Before; @@ -22,12 +20,7 @@ * Basic tests for simple xpack functionality that are only run if the * cluster is the on the default distribution. */ -public class XPackIT extends ParameterizedRollingUpgradeTestCase { - - public XPackIT(@Name("upgradeNode") Integer upgradeNode) { - super(upgradeNode); - } - +public class XPackIT extends AbstractRollingTestCase { @Before public void skipIfNotXPack() { assumeThat( @@ -35,9 +28,10 @@ public void skipIfNotXPack() { System.getProperty("tests.distribution"), equalTo("default") ); - assumeTrue( + assumeThat( "running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway", - isUpgradedCluster() + CLUSTER_TYPE, + equalTo(ClusterType.UPGRADED) ); /* * *Mostly* we want this for when we're upgrading from pre-6.3's diff --git a/qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml similarity index 100% rename from qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml diff --git a/qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/mixed_cluster/20_camel_case_on_format.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/20_camel_case_on_format.yml similarity index 100% rename from qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/mixed_cluster/20_camel_case_on_format.yml rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/20_camel_case_on_format.yml diff --git a/qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/mixed_cluster/30_vector_search.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_vector_search.yml similarity index 100% rename from qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/mixed_cluster/30_vector_search.yml rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_vector_search.yml diff --git a/qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml similarity index 100% rename from qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml diff --git a/qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/old_cluster/20_camel_case_on_format.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_camel_case_on_format.yml similarity index 100% rename from qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/old_cluster/20_camel_case_on_format.yml rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_camel_case_on_format.yml diff --git a/qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml similarity index 99% rename from qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml index b471fa56a47a5..11e9fdc2cca95 100644 --- a/qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml @@ -11,6 +11,7 @@ bdv: type: dense_vector dims: 3 + index: false knn: type: dense_vector dims: 3 @@ -125,6 +126,7 @@ bdv: type: dense_vector element_type: byte + index: false dims: 3 knn: type: dense_vector diff --git a/qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml similarity index 100% rename from qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml diff --git a/qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/upgraded_cluster/20_camel_case_on_format.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_camel_case_on_format.yml similarity index 100% rename from qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/upgraded_cluster/20_camel_case_on_format.yml rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_camel_case_on_format.yml diff --git a/qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/upgraded_cluster/30_vector_search.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_vector_search.yml similarity index 100% rename from qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/upgraded_cluster/30_vector_search.yml rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_vector_search.yml From a33ebc6f03f949408d87bd1ca81044f822793d50 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 21 Sep 2023 15:18:27 +0100 Subject: [PATCH 06/16] Migrate IndexShardOperationPermits to proper executor (#99719) Relates #99392, #97879 --- .../TransportReplicationAction.java | 6 +- .../org/elasticsearch/index/IndexService.java | 3 +- .../index/seqno/RetentionLeaseActions.java | 3 +- .../elasticsearch/index/shard/IndexShard.java | 24 +++--- .../shard/IndexShardOperationPermits.java | 27 +++++-- .../recovery/RecoverySourceHandler.java | 2 +- ...TransportResyncReplicationActionTests.java | 5 +- .../TransportReplicationActionTests.java | 7 +- .../TransportWriteActionTests.java | 5 +- .../IndexShardOperationPermitsTests.java | 76 ++++++++----------- .../index/shard/IndexShardTests.java | 46 ++++++----- .../recovery/RecoverySourceHandlerTests.java | 7 +- .../ESIndexLevelReplicationTestCase.java | 11 +-- .../index/shard/IndexShardTestCase.java | 3 + .../action/TransportForgetFollowerAction.java | 3 +- .../ShardFollowTaskReplicationTests.java | 7 +- .../engine/FollowEngineIndexShardTests.java | 3 +- 17 files changed, 128 insertions(+), 110 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index d1e2b6dd98faa..b3f0496597009 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -117,7 +117,7 @@ public abstract class TransportReplicationAction< protected final ShardStateAction shardStateAction; protected final IndicesService indicesService; protected final TransportRequestOptions transportOptions; - protected final String executor; + protected final Executor executor; protected final boolean forceExecutionOnPrimary; // package private for testing @@ -179,7 +179,7 @@ protected TransportReplicationAction( this.clusterService = clusterService; this.indicesService = indicesService; this.shardStateAction = shardStateAction; - this.executor = executor; + this.executor = threadPool.executor(executor); this.transportPrimaryAction = actionName + "[p]"; this.transportReplicaAction = actionName + "[r]"; @@ -272,7 +272,7 @@ protected abstract void shardOperationOnPrimary( /** * Execute the specified replica operation. This is done under a permit from - * {@link IndexShard#acquireReplicaOperationPermit(long, long, long, ActionListener, String)}. + * {@link IndexShard#acquireReplicaOperationPermit(long, long, long, ActionListener, Executor)}. * * @param shardRequest the request to the replica shard * @param replica the replica shard to perform the operation on diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 05c6fd63c3fcb..781ff035a79ff 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.concurrent.AbstractAsyncTask; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.IOUtils; @@ -1047,7 +1048,7 @@ private void sync(final Consumer sync, final String source) { && e instanceof ShardNotInPrimaryModeException == false) { logger.warn(() -> format("%s failed to execute %s sync", shard.shardId(), source), e); } - }, ThreadPool.Names.SAME); + }, EsExecutors.DIRECT_EXECUTOR_SERVICE); } catch (final AlreadyClosedException | IndexShardClosedException e) { // the shard was closed concurrently, continue } diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java index 7393371788f8c..2866d1b0ee13a 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.IndexShard; @@ -86,7 +87,7 @@ protected void asyncShardOperation(T request, ShardId shardId, final ActionListe try (Releasable ignore = releasable) { doRetentionLeaseAction(indexShard, request, delegatedListener); } - }), ThreadPool.Names.SAME); + }), EsExecutors.DIRECT_EXECUTOR_SERVICE); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 3adee7643990b..c168f3d64f7ea 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -839,8 +839,8 @@ public void onFailure(Exception e) { listener.onFailure(e); } } - }, 30L, TimeUnit.MINUTES, ThreadPool.Names.SAME); // Wait on SAME (current thread) because this execution is wrapped by - // CancellableThreads and we want to be able to safely interrupt it + }, 30L, TimeUnit.MINUTES, EsExecutors.DIRECT_EXECUTOR_SERVICE); // Wait on current thread because this execution is wrapped by + // CancellableThreads and we want to be able to interrupt it } } @@ -3361,11 +3361,15 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) { * ActionListener will then be called using the provided executor. * */ - public void acquirePrimaryOperationPermit(ActionListener onPermitAcquired, String executorOnDelay) { + public void acquirePrimaryOperationPermit(ActionListener onPermitAcquired, Executor executorOnDelay) { acquirePrimaryOperationPermit(onPermitAcquired, executorOnDelay, false); } - public void acquirePrimaryOperationPermit(ActionListener onPermitAcquired, String executorOnDelay, boolean forceExecution) { + public void acquirePrimaryOperationPermit( + ActionListener onPermitAcquired, + Executor executorOnDelay, + boolean forceExecution + ) { verifyNotClosed(); assert shardRouting.primary() : "acquirePrimaryOperationPermit should only be called on primary shard: " + shardRouting; indexShardOperationPermits.acquire(wrapPrimaryOperationPermitListener(onPermitAcquired), executorOnDelay, forceExecution); @@ -3415,7 +3419,7 @@ private void asyncBlockOperations(ActionListener onPermitAcquired, l onPermitAcquired.onFailure(e); }); try { - indexShardOperationPermits.blockOperations(wrappedListener, timeout, timeUnit, ThreadPool.Names.GENERIC); + indexShardOperationPermits.blockOperations(wrappedListener, timeout, timeUnit, threadPool.generic()); } catch (Exception e) { forceRefreshes.close(); throw e; @@ -3424,7 +3428,7 @@ private void asyncBlockOperations(ActionListener onPermitAcquired, l /** * Runs the specified runnable under a permit and otherwise calling back the specified failure callback. This method is really a - * convenience for {@link #acquirePrimaryOperationPermit(ActionListener, String)} where the listener equates to + * convenience for {@link #acquirePrimaryOperationPermit(ActionListener, Executor)} where the listener equates to * try-with-resources closing the releasable after executing the runnable on successfully acquiring the permit, an otherwise calling * back the failure callback. * @@ -3432,7 +3436,7 @@ private void asyncBlockOperations(ActionListener onPermitAcquired, l * @param onFailure the callback on failure * @param executorOnDelay the executor to execute the runnable on if permit acquisition is blocked */ - public void runUnderPrimaryPermit(final Runnable runnable, final Consumer onFailure, final String executorOnDelay) { + public void runUnderPrimaryPermit(final Runnable runnable, final Consumer onFailure, final Executor executorOnDelay) { verifyNotClosed(); assert shardRouting.primary() : "runUnderPrimaryPermit should only be called on primary shard but was " + shardRouting; final ActionListener onPermitAcquired = ActionListener.wrap(releasable -> { @@ -3505,7 +3509,7 @@ public void onResponse(final Releasable releasable) { /** * Acquire a replica operation permit whenever the shard is ready for indexing (see - * {@link #acquirePrimaryOperationPermit(ActionListener, String)}). If the given primary term is lower than then one in + * {@link #acquirePrimaryOperationPermit(ActionListener, Executor)}). If the given primary term is lower than then one in * {@link #shardRouting}, the {@link ActionListener#onFailure(Exception)} method of the provided listener is invoked with an * {@link IllegalStateException}. If permit acquisition is delayed, the listener will be invoked on the executor with the specified * name. @@ -3522,7 +3526,7 @@ public void acquireReplicaOperationPermit( final long globalCheckpoint, final long maxSeqNoOfUpdatesOrDeletes, final ActionListener onPermitAcquired, - final String executorOnDelay + final Executor executorOnDelay ) { innerAcquireReplicaOperationPermit( opPrimaryTerm, @@ -4138,7 +4142,7 @@ public long getMaxSeqNoOfUpdatesOrDeletes() { * These transfers guarantee that every index/delete operation when executing on a replica engine will observe this marker a value * which is at least the value of the max_seq_no_of_updates marker on the primary after that operation was executed on the primary. * - * @see #acquireReplicaOperationPermit(long, long, long, ActionListener, String) + * @see #acquireReplicaOperationPermit(long, long, long, ActionListener, Executor) * @see RecoveryTarget#indexTranslogOperations(List, int, long, long, RetentionLeases, long, ActionListener) */ public void advanceMaxSeqNoOfUpdatesOrDeletes(long seqNo) { diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java index 9c4cd3958936d..9da78953d7213 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; @@ -27,6 +28,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.Executor; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -77,13 +79,18 @@ public void close() { * @param timeUnit the time unit of the {@code timeout} argument * @param executor executor on which to wait for in-flight operations to finish and acquire all permits */ - public void blockOperations(final ActionListener onAcquired, final long timeout, final TimeUnit timeUnit, String executor) { + public void blockOperations( + final ActionListener onAcquired, + final long timeout, + final TimeUnit timeUnit, + final Executor executor + ) { delayOperations(); waitUntilBlocked(ActionListener.assertOnce(onAcquired), timeout, timeUnit, executor); } - private void waitUntilBlocked(ActionListener onAcquired, long timeout, TimeUnit timeUnit, String executor) { - threadPool.executor(executor).execute(new AbstractRunnable() { + private void waitUntilBlocked(ActionListener onAcquired, long timeout, TimeUnit timeUnit, Executor executor) { + executor.execute(new AbstractRunnable() { final Releasable released = Releasables.releaseOnce(() -> releaseDelayedOperations()); @@ -187,11 +194,19 @@ private void releaseDelayedOperations() { * @param executorOnDelay executor to use for the possibly delayed {@link ActionListener#onResponse(Object)} call * @param forceExecution whether the runnable should force its execution in case it gets rejected */ - public void acquire(final ActionListener onAcquired, final String executorOnDelay, final boolean forceExecution) { + public void acquire( + final ActionListener onAcquired, + @Nullable final Executor executorOnDelay, + final boolean forceExecution + ) { innerAcquire(ActionListener.assertOnce(onAcquired), executorOnDelay, forceExecution); } - private void innerAcquire(final ActionListener onAcquired, final String executorOnDelay, final boolean forceExecution) { + private void innerAcquire( + final ActionListener onAcquired, + @Nullable final Executor executorOnDelay, + final boolean forceExecution + ) { if (closed) { onAcquired.onFailure(new IndexShardClosedException(shardId)); return; @@ -204,7 +219,7 @@ private void innerAcquire(final ActionListener onAcquired, final Str final ActionListener wrappedListener; if (executorOnDelay != null) { wrappedListener = new ContextPreservingActionListener<>(contextSupplier, onAcquired).delegateFailure( - (l, r) -> threadPool.executor(executorOnDelay).execute(new ActionRunnable<>(l) { + (l, r) -> executorOnDelay.execute(new ActionRunnable<>(l) { @Override public boolean isForceExecution() { return forceExecution; diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index 1acc0f1041e39..fc5df1a4aa282 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -422,7 +422,7 @@ public void onFailure(Exception e) { cancellableThreads.checkForCancel(); ensureNotRelocatedPrimary(primary); action.accept(l2); - })), ThreadPool.Names.GENERIC); + })), primary.getThreadPool().generic()); } static void runUnderPrimaryPermit( diff --git a/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java index b41f0e667966c..f3cf4edfb8bec 100644 --- a/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java @@ -50,6 +50,7 @@ import java.util.Collections; import java.util.HashSet; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -62,7 +63,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -147,7 +148,7 @@ public void testResyncDoesNotBlockOnPrimaryAction() throws Exception { acquiredPermits.incrementAndGet(); callback.onResponse(acquiredPermits::decrementAndGet); return null; - }).when(indexShard).acquirePrimaryOperationPermit(anyActionListener(), anyString(), eq(true)); + }).when(indexShard).acquirePrimaryOperationPermit(anyActionListener(), any(Executor.class), eq(true)); when(indexShard.getReplicationGroup()).thenReturn( new ReplicationGroup( shardRoutingTable, diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 09d22bd00f602..308c8a66767fa 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -91,6 +91,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -944,7 +945,7 @@ public void testSeqNoIsSetOnPrimary() { ActionListener argument = (ActionListener) invocation.getArguments()[0]; argument.onResponse(count::decrementAndGet); return null; - }).when(shard).acquirePrimaryOperationPermit(any(), anyString(), eq(forceExecute)); + }).when(shard).acquirePrimaryOperationPermit(any(), any(Executor.class), eq(forceExecute)); when(shard.getActiveOperationsCount()).thenAnswer(i -> count.get()); final IndexService indexService = mock(IndexService.class); @@ -1580,7 +1581,7 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService callback.onFailure(new ShardNotInPrimaryModeException(shardId, IndexShardState.STARTED)); } return null; - }).when(indexShard).acquirePrimaryOperationPermit(any(ActionListener.class), anyString(), eq(forceExecute)); + }).when(indexShard).acquirePrimaryOperationPermit(any(ActionListener.class), any(Executor.class), eq(forceExecute)); when(indexShard.isPrimaryMode()).thenAnswer(invocation -> isPrimaryMode.get()); doAnswer(invocation -> { long term = (Long) invocation.getArguments()[0]; @@ -1594,7 +1595,7 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService count.incrementAndGet(); callback.onResponse(count::decrementAndGet); return null; - }).when(indexShard).acquireReplicaOperationPermit(anyLong(), anyLong(), anyLong(), any(ActionListener.class), anyString()); + }).when(indexShard).acquireReplicaOperationPermit(anyLong(), anyLong(), anyLong(), any(ActionListener.class), any(Executor.class)); when(indexShard.getActiveOperationsCount()).thenAnswer(i -> count.get()); when(indexShard.routingEntry()).thenAnswer(invocationOnMock -> { diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java index 7726feaa30868..4f926b14a1904 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -61,6 +61,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -545,7 +546,7 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService count.incrementAndGet(); callback.onResponse(count::decrementAndGet); return null; - }).when(indexShard).acquirePrimaryOperationPermit(anyActionListener(), anyString(), any()); + }).when(indexShard).acquirePrimaryOperationPermit(anyActionListener(), any(Executor.class), any()); doAnswer(invocation -> { long term = (Long) invocation.getArguments()[0]; @SuppressWarnings("unchecked") @@ -559,7 +560,7 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService count.incrementAndGet(); callback.onResponse(count::decrementAndGet); return null; - }).when(indexShard).acquireReplicaOperationPermit(anyLong(), anyLong(), anyLong(), anyActionListener(), anyString()); + }).when(indexShard).acquireReplicaOperationPermit(anyLong(), anyLong(), anyLong(), anyActionListener(), any(Executor.class)); when(indexShard.routingEntry()).thenAnswer(invocationOnMock -> { final ClusterState state = clusterService.state(); final RoutingNode node = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java index 240e0b9abd618..c6f0df53d68ff 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java @@ -36,6 +36,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -110,7 +111,7 @@ public void testAllOperationsInvoked() throws InterruptedException, TimeoutExcep boolean forceExecution = randomBoolean(); for (int i = 0; i < numThreads; i++) { // the write thread pool uses a bounded size and can get rejections, see setupThreadPool - String threadPoolName = randomFrom(ThreadPool.Names.WRITE, ThreadPool.Names.GENERIC); + Executor executor = threadPool.executor(randomFrom(ThreadPool.Names.WRITE, ThreadPool.Names.GENERIC)); PlainActionFuture future = new PlainActionFuture() { @Override public void onResponse(Releasable releasable) { @@ -120,7 +121,7 @@ public void onResponse(Releasable releasable) { }; Thread thread = new Thread(() -> { latch.countDown(); - permits.acquire(future, threadPoolName, forceExecution); + permits.acquire(future, executor, forceExecution); }); futures.add(future); operationThreads.add(thread); @@ -169,7 +170,7 @@ public void onResponse(Releasable releasable) { public void testOperationsInvokedImmediatelyIfNoBlock() throws ExecutionException, InterruptedException { PlainActionFuture future = new PlainActionFuture<>(); - permits.acquire(future, ThreadPool.Names.GENERIC, true); + permits.acquire(future, threadPool.generic(), true); assertTrue(future.isDone()); future.get().close(); } @@ -177,7 +178,7 @@ public void testOperationsInvokedImmediatelyIfNoBlock() throws ExecutionExceptio public void testOperationsIfClosed() { PlainActionFuture future = new PlainActionFuture<>(); permits.close(); - permits.acquire(future, ThreadPool.Names.GENERIC, true); + permits.acquire(future, threadPool.generic(), true); ExecutionException exception = expectThrows(ExecutionException.class, future::get); assertThat(exception.getCause(), instanceOf(IndexShardClosedException.class)); } @@ -190,7 +191,7 @@ public void testBlockIfClosed() { wrap(() -> { throw new IllegalArgumentException("fake error"); }), randomInt(10), TimeUnit.MINUTES, - ThreadPool.Names.GENERIC + threadPool.generic() ) ); } @@ -198,7 +199,7 @@ public void testBlockIfClosed() { public void testOperationsDelayedIfBlock() throws ExecutionException, InterruptedException, TimeoutException { PlainActionFuture future = new PlainActionFuture<>(); try (Releasable ignored = blockAndWait()) { - permits.acquire(future, ThreadPool.Names.GENERIC, true); + permits.acquire(future, threadPool.generic(), true); assertFalse(future.isDone()); } future.get(1, TimeUnit.HOURS).close(); @@ -210,13 +211,13 @@ public void testGetBlockWhenBlocked() throws ExecutionException, InterruptedExce final CountDownLatch releaseBlock = new CountDownLatch(1); final AtomicBoolean blocked = new AtomicBoolean(); try (Releasable ignored = blockAndWait()) { - permits.acquire(future, ThreadPool.Names.GENERIC, true); + permits.acquire(future, threadPool.generic(), true); permits.blockOperations(wrap(() -> { blocked.set(true); blockAcquired.countDown(); releaseBlock.await(); - }), 30, TimeUnit.MINUTES, ThreadPool.Names.GENERIC); + }), 30, TimeUnit.MINUTES, threadPool.generic()); assertFalse(blocked.get()); assertFalse(future.isDone()); } @@ -275,7 +276,7 @@ public void onResponse(Releasable releasable) { context.putHeader("foo", "bar"); context.putTransient("bar", "baz"); // test both with and without a executor name - permits.acquire(future, ThreadPool.Names.GENERIC, true); + permits.acquire(future, threadPool.generic(), true); permits.acquire(future2, null, true); } assertFalse(future.isDone()); @@ -310,7 +311,7 @@ public void onFailure(Exception e) { throw new RuntimeException(e); } } - }, blockReleased::countDown), 1, TimeUnit.MINUTES, ThreadPool.Names.GENERIC); + }, blockReleased::countDown), 1, TimeUnit.MINUTES, threadPool.generic()); blockAcquired.await(); return () -> { releaseBlock.countDown(); @@ -330,7 +331,7 @@ public void testAsyncBlockOperationsOperationWhileBlocked() throws InterruptedEx blocked.set(true); blockAcquired.countDown(); releaseBlock.await(); - }), 30, TimeUnit.MINUTES, ThreadPool.Names.GENERIC); + }), 30, TimeUnit.MINUTES, threadPool.generic()); blockAcquired.await(); assertTrue(blocked.get()); @@ -349,7 +350,7 @@ public void onResponse(Releasable releasable) { public void onFailure(Exception e) { } - }, ThreadPool.Names.GENERIC, false)); + }, threadPool.generic(), false)); thread.start(); assertFalse(delayed.get()); releaseBlock.countDown(); @@ -378,7 +379,7 @@ public void testAsyncBlockOperationsOperationBeforeBlocked() throws InterruptedE permits.blockOperations(wrap(() -> { onBlocked.set(true); blockedLatch.countDown(); - }), 30, TimeUnit.MINUTES, ThreadPool.Names.GENERIC); + }), 30, TimeUnit.MINUTES, threadPool.generic()); assertFalse(onBlocked.get()); // if we submit another operation, it should be delayed @@ -399,7 +400,7 @@ public void onResponse(Releasable releasable) { public void onFailure(Exception e) { throw new RuntimeException(e); } - }, ThreadPool.Names.GENERIC, false); + }, threadPool.generic(), false); }); secondOperationThread.start(); @@ -445,7 +446,7 @@ public void onResponse(Releasable releasable) { public void onFailure(Exception e) { } - }, ThreadPool.Names.GENERIC, false); + }, threadPool.generic(), false); }); thread.start(); threads.add(thread); @@ -460,7 +461,7 @@ public void onFailure(Exception e) { permits.blockOperations(wrap(() -> { values.add(operations); operationLatch.countDown(); - }), 30, TimeUnit.MINUTES, ThreadPool.Names.GENERIC); + }), 30, TimeUnit.MINUTES, threadPool.generic()); }); blockingThread.start(); @@ -495,12 +496,12 @@ public void onFailure(Exception e) { public void testActiveOperationsCount() throws ExecutionException, InterruptedException { PlainActionFuture future1 = new PlainActionFuture<>(); - permits.acquire(future1, ThreadPool.Names.GENERIC, true); + permits.acquire(future1, threadPool.generic(), true); assertTrue(future1.isDone()); assertThat(permits.getActiveOperationsCount(), equalTo(1)); PlainActionFuture future2 = new PlainActionFuture<>(); - permits.acquire(future2, ThreadPool.Names.GENERIC, true); + permits.acquire(future2, threadPool.generic(), true); assertTrue(future2.isDone()); assertThat(permits.getActiveOperationsCount(), equalTo(2)); @@ -516,7 +517,7 @@ public void testActiveOperationsCount() throws ExecutionException, InterruptedEx } PlainActionFuture future3 = new PlainActionFuture<>(); - permits.acquire(future3, ThreadPool.Names.GENERIC, true); + permits.acquire(future3, threadPool.generic(), true); assertTrue(future3.isDone()); assertThat(permits.getActiveOperationsCount(), equalTo(1)); future3.get().close(); @@ -525,26 +526,21 @@ public void testActiveOperationsCount() throws ExecutionException, InterruptedEx public void testAsyncBlockOperationsOnRejection() { final PlainActionFuture threadBlock = new PlainActionFuture<>(); - try ( - Releasable firstPermit = PlainActionFuture.get(f -> permits.acquire(f, ThreadPool.Names.GENERIC, false), 0, TimeUnit.SECONDS) - ) { + try (Releasable firstPermit = PlainActionFuture.get(f -> permits.acquire(f, threadPool.generic(), false), 0, TimeUnit.SECONDS)) { assertNotNull(firstPermit); - threadPool.executor(REJECTING_EXECUTOR).execute(threadBlock::actionGet); + final var rejectingExecutor = threadPool.executor(REJECTING_EXECUTOR); + rejectingExecutor.execute(threadBlock::actionGet); expectThrows( EsRejectedExecutionException.class, () -> PlainActionFuture.get( - f -> permits.blockOperations(f, 1, TimeUnit.HOURS, REJECTING_EXECUTOR) + f -> permits.blockOperations(f, 1, TimeUnit.HOURS, rejectingExecutor) ) ); // ensure that the exception means no block was put in place try ( - Releasable secondPermit = PlainActionFuture.get( - f -> permits.acquire(f, ThreadPool.Names.GENERIC, false), - 0, - TimeUnit.SECONDS - ) + Releasable secondPermit = PlainActionFuture.get(f -> permits.acquire(f, threadPool.generic(), false), 0, TimeUnit.SECONDS) ) { assertNotNull(secondPermit); } @@ -553,16 +549,14 @@ public void testAsyncBlockOperationsOnRejection() { } // ensure that another block can still be acquired - try (Releasable block = PlainActionFuture.get(f -> permits.blockOperations(f, 1, TimeUnit.HOURS, ThreadPool.Names.GENERIC))) { + try (Releasable block = PlainActionFuture.get(f -> permits.blockOperations(f, 1, TimeUnit.HOURS, threadPool.generic()))) { assertNotNull(block); } } public void testAsyncBlockOperationsOnTimeout() { final PlainActionFuture threadBlock = new PlainActionFuture<>(); - try ( - Releasable firstPermit = PlainActionFuture.get(f -> permits.acquire(f, ThreadPool.Names.GENERIC, false), 0, TimeUnit.SECONDS) - ) { + try (Releasable firstPermit = PlainActionFuture.get(f -> permits.acquire(f, threadPool.generic(), false), 0, TimeUnit.SECONDS)) { assertNotNull(firstPermit); assertEquals( @@ -570,18 +564,14 @@ public void testAsyncBlockOperationsOnTimeout() { expectThrows( ElasticsearchTimeoutException.class, () -> PlainActionFuture.get( - f -> permits.blockOperations(f, 0, TimeUnit.SECONDS, ThreadPool.Names.GENERIC) + f -> permits.blockOperations(f, 0, TimeUnit.SECONDS, threadPool.generic()) ) ).getMessage() ); // ensure that the exception means no block was put in place try ( - Releasable secondPermit = PlainActionFuture.get( - f -> permits.acquire(f, ThreadPool.Names.GENERIC, false), - 0, - TimeUnit.SECONDS - ) + Releasable secondPermit = PlainActionFuture.get(f -> permits.acquire(f, threadPool.generic(), false), 0, TimeUnit.SECONDS) ) { assertNotNull(secondPermit); } @@ -591,7 +581,7 @@ public void testAsyncBlockOperationsOnTimeout() { } // ensure that another block can still be acquired - try (Releasable block = PlainActionFuture.get(f -> permits.blockOperations(f, 1, TimeUnit.HOURS, ThreadPool.Names.GENERIC))) { + try (Releasable block = PlainActionFuture.get(f -> permits.blockOperations(f, 1, TimeUnit.HOURS, threadPool.generic()))) { assertNotNull(block); } } @@ -622,7 +612,7 @@ public void onFailure(final Exception e) { reference.set(e); onFailureLatch.countDown(); } - }, 1, TimeUnit.MILLISECONDS, ThreadPool.Names.GENERIC); + }, 1, TimeUnit.MILLISECONDS, threadPool.generic()); onFailureLatch.await(); assertThat(reference.get(), hasToString(containsString("timeout while blocking operations"))); @@ -647,7 +637,7 @@ public void onResponse(Releasable releasable) { public void onFailure(Exception e) { assert false; } - }, ThreadPool.Names.GENERIC, false) + }, threadPool.generic(), false) ); assertThat(e, hasToString(containsString("failed to obtain permit but operations are not delayed"))); permits.semaphore.release(IndexShardOperationPermits.TOTAL_PERMITS); @@ -697,7 +687,7 @@ public void onResponse(Releasable releasable) { public void onFailure(Exception e) { throw new RuntimeException(e); } - }, ThreadPool.Names.GENERIC, false); + }, threadPool.generic(), false); }; } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 5fdfb92d3a193..4850cd4f4cf5e 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -60,6 +60,7 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ReleasableLock; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.CheckedFunction; @@ -150,6 +151,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -330,7 +332,7 @@ public void testClosesPreventsNewOperations() throws Exception { IndexShard indexShard = newStartedShard(); closeShards(indexShard); assertThat(indexShard.getActiveOperationsCount(), equalTo(0)); - expectThrows(IndexShardClosedException.class, () -> indexShard.acquirePrimaryOperationPermit(null, ThreadPool.Names.WRITE)); + expectThrows(IndexShardClosedException.class, () -> indexShard.acquirePrimaryOperationPermit(null, writeExecutor)); expectThrows( IndexShardClosedException.class, () -> indexShard.acquireAllPrimaryOperationsPermits(null, TimeValue.timeValueSeconds(30L)) @@ -342,7 +344,7 @@ public void testClosesPreventsNewOperations() throws Exception { UNASSIGNED_SEQ_NO, randomNonNegativeLong(), null, - ThreadPool.Names.WRITE + writeExecutor ) ); expectThrows( @@ -364,7 +366,7 @@ public void testRunUnderPrimaryPermitRunsUnderPrimaryPermit() throws IOException indexShard.runUnderPrimaryPermit( () -> assertThat(indexShard.getActiveOperationsCount(), equalTo(1)), e -> fail(e.toString()), - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); assertThat(indexShard.getActiveOperationsCount(), equalTo(0)); } finally { @@ -380,7 +382,7 @@ public void testRunUnderPrimaryPermitOnFailure() throws IOException { assertThat(e, instanceOf(RuntimeException.class)); assertThat(e.getMessage(), equalTo("failure")); invoked.set(true); - }, ThreadPool.Names.SAME); + }, EsExecutors.DIRECT_EXECUTOR_SERVICE); assertTrue(invoked.get()); } finally { closeShards(indexShard); @@ -406,7 +408,7 @@ public void testRunUnderPrimaryPermitDelaysToExecutorWhenBlocked() throws Except : executorOnDelay.toLowerCase(Locale.ROOT); assertThat(Thread.currentThread().getName(), containsString(expectedThreadPoolName)); latch.countDown(); - }, e -> fail(e.toString()), executorOnDelay); + }, e -> fail(e.toString()), threadPool.executor(executorOnDelay)); permit.close(); latch.await(); // we could race and assert on the count before the permit is returned @@ -469,7 +471,7 @@ public void onFailure(Exception e) { throw new RuntimeException(e); } }, - ThreadPool.Names.WRITE + writeExecutor ); }); thread.start(); @@ -513,7 +515,7 @@ public void onResponse(Releasable releasable) { public void onFailure(Exception e) { throw new RuntimeException(e); } - }, ThreadPool.Names.WRITE); + }, writeExecutor); }); thread.start(); delayedThreads.add(thread); @@ -559,7 +561,7 @@ public void testPublishingOrderOnPromotion() throws IOException, InterruptedExce if (indexShard.routingEntry().primary()) { assertThat(indexShard.getPendingPrimaryTerm(), equalTo(promotedTerm)); final PlainActionFuture permitAcquiredFuture = new PlainActionFuture<>(); - indexShard.acquirePrimaryOperationPermit(permitAcquiredFuture, ThreadPool.Names.SAME); + indexShard.acquirePrimaryOperationPermit(permitAcquiredFuture, EsExecutors.DIRECT_EXECUTOR_SERVICE); try (Releasable ignored = permitAcquiredFuture.actionGet()) { assertThat(indexShard.getReplicationGroup(), notNullValue()); } @@ -619,7 +621,7 @@ public void onResponse(Releasable releasable) { public void onFailure(Exception e) { throw new AssertionError(e); } - }, ThreadPool.Names.GENERIC); + }, threadPool.generic()); latch.await(); assertThat(indexShard.getLocalCheckpoint(), equalTo((long) maxSeqNo)); @@ -667,7 +669,7 @@ public void onResponse(Releasable releasable) { public void onFailure(Exception e) { throw new RuntimeException(e); } - }, ThreadPool.Names.GENERIC); + }, threadPool.generic()); latch.await(); assertThat(getTranslog(indexShard).getGeneration().translogFileGeneration, equalTo(currentTranslogGeneration + 1)); @@ -750,7 +752,7 @@ public void onFailure(final Exception e) { assertThat(e, instanceOf(ShardNotInPrimaryModeException.class)); assertThat(e, hasToString(containsString("shard is not in primary mode"))); } - }, ThreadPool.Names.SAME); + }, EsExecutors.DIRECT_EXECUTOR_SERVICE); final CountDownLatch latch = new CountDownLatch(1); indexShard.acquireAllPrimaryOperationsPermits(new ActionListener<>() { @@ -788,7 +790,7 @@ public void onFailure(Exception e) { fail(); } }, - ThreadPool.Names.WRITE + writeExecutor ) ).getMessage(), containsString("in primary mode cannot be a replication target") @@ -842,7 +844,7 @@ public void onFailure(final Exception e) { throw new RuntimeException(e); } if (singlePermit) { - indexShard.acquirePrimaryOperationPermit(future, ThreadPool.Names.WRITE); + indexShard.acquirePrimaryOperationPermit(future, writeExecutor); } else { indexShard.acquireAllPrimaryOperationsPermits(future, TimeValue.timeValueHours(1L)); } @@ -899,7 +901,7 @@ public void onResponse(final Releasable releasable) { private Releasable acquirePrimaryOperationPermitBlockingly(IndexShard indexShard) throws ExecutionException, InterruptedException { PlainActionFuture fut = new PlainActionFuture<>(); - indexShard.acquirePrimaryOperationPermit(fut, ThreadPool.Names.WRITE); + indexShard.acquirePrimaryOperationPermit(fut, writeExecutor); return fut.get(); } @@ -911,7 +913,7 @@ private Releasable acquireReplicaOperationPermitBlockingly(IndexShard indexShard indexShard.getLastKnownGlobalCheckpoint(), randomNonNegativeLong(), fut, - ThreadPool.Names.WRITE + writeExecutor ); return fut.get(); } @@ -963,10 +965,7 @@ public void testOperationPermitOnReplicaShards() throws Exception { assertEquals(0, indexShard.getActiveOperationsCount()); if (shardRouting.primary() == false && Assertions.ENABLED) { - AssertionError e = expectThrows( - AssertionError.class, - () -> indexShard.acquirePrimaryOperationPermit(null, ThreadPool.Names.WRITE) - ); + AssertionError e = expectThrows(AssertionError.class, () -> indexShard.acquirePrimaryOperationPermit(null, writeExecutor)); assertThat(e, hasToString(containsString("acquirePrimaryOperationPermit should only be called on primary shard"))); e = expectThrows( @@ -1424,7 +1423,7 @@ public void onFailure(Exception e) { latch.countDown(); } }, - ThreadPool.Names.WRITE + writeExecutor ); }; @@ -1999,7 +1998,7 @@ public void onFailure(Exception e) { }); } - shard.acquirePrimaryOperationPermit(onLockAcquired, ThreadPool.Names.WRITE); + shard.acquirePrimaryOperationPermit(onLockAcquired, writeExecutor); } for (final Runnable assertion : assertions) { @@ -4539,7 +4538,7 @@ public void testTypelessGet() throws IOException { } /** - * Randomizes the usage of {@link IndexShard#acquireReplicaOperationPermit(long, long, long, ActionListener, String)} and + * Randomizes the usage of {@link IndexShard#acquireReplicaOperationPermit(long, long, long, ActionListener, Executor)} and * {@link IndexShard#acquireAllReplicaOperationsPermits(long, long, long, ActionListener, TimeValue)} in order to acquire a permit. */ private void randomReplicaOperationPermitAcquisition( @@ -4550,8 +4549,7 @@ private void randomReplicaOperationPermitAcquisition( final ActionListener listener ) { if (randomBoolean()) { - final String executor = ThreadPool.Names.WRITE; - indexShard.acquireReplicaOperationPermit(opPrimaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, listener, executor); + indexShard.acquireReplicaOperationPermit(opPrimaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, listener, writeExecutor); } else { final TimeValue timeout = TimeValue.timeValueSeconds(30L); indexShard.acquireAllReplicaOperationsPermits(opPrimaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, listener, timeout); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index d2f94ff2d344a..0285a8c66fffb 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -124,7 +124,6 @@ import static org.hamcrest.Matchers.notNullValue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -709,6 +708,7 @@ public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Started() throws IOE final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service); final StartRecoveryRequest request = getStartRecoveryRequest(); final IndexShard shard = mock(IndexShard.class); + when(shard.getThreadPool()).thenReturn(threadPool); when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class)); when(shard.segmentStats(anyBoolean(), anyBoolean())).thenReturn(mock(SegmentsStats.class)); when(shard.isRelocatedPrimary()).thenReturn(true); @@ -716,7 +716,7 @@ public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Started() throws IOE doAnswer(invocation -> { ((ActionListener) invocation.getArguments()[0]).onResponse(() -> {}); return null; - }).when(shard).acquirePrimaryOperationPermit(any(), anyString()); + }).when(shard).acquirePrimaryOperationPermit(any(), any(Executor.class)); final IndexMetadata.Builder indexMetadata = IndexMetadata.builder("test") .settings( @@ -799,11 +799,12 @@ public void testCancellationsDoesNotLeakPrimaryPermits() throws Exception { final IndexShard shard = mock(IndexShard.class); final AtomicBoolean freed = new AtomicBoolean(true); when(shard.isRelocatedPrimary()).thenReturn(false); + when(shard.getThreadPool()).thenReturn(threadPool); doAnswer(invocation -> { freed.set(false); ((ActionListener) invocation.getArguments()[0]).onResponse(() -> freed.set(true)); return null; - }).when(shard).acquirePrimaryOperationPermit(any(), anyString()); + }).when(shard).acquirePrimaryOperationPermit(any(), any(Executor.class)); Thread cancelingThread = new Thread(() -> cancellableThreads.cancel("test")); cancelingThread.start(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index c2ce750f155cf..a05b2510bc056 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -52,6 +52,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; @@ -619,7 +620,7 @@ public void executeRetentionLeasesSyncRequestOnReplica(RetentionLeaseSyncAction. getPrimary().getLastKnownGlobalCheckpoint(), getPrimary().getMaxSeqNoOfUpdatesOrDeletes(), acquirePermitFuture, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); try (Releasable ignored = acquirePermitFuture.actionGet()) { replica.updateRetentionLeasesOnReplica(request.getRetentionLeases()); @@ -792,7 +793,7 @@ public void performOn( delegatedListener.onFailure(e); } }), - ThreadPool.Names.WRITE + replica.getThreadPool().executor(ThreadPool.Names.WRITE) ); } @@ -883,7 +884,7 @@ private void executeShardBulkOnPrimary( } } final PlainActionFuture permitAcquiredFuture = new PlainActionFuture<>(); - primary.acquirePrimaryOperationPermit(permitAcquiredFuture, ThreadPool.Names.SAME); + primary.acquirePrimaryOperationPermit(permitAcquiredFuture, EsExecutors.DIRECT_EXECUTOR_SERVICE); try (Releasable ignored = permitAcquiredFuture.actionGet()) { MappingUpdatePerformer noopMappingUpdater = (_update, _shardId, _listener1) -> {}; TransportShardBulkAction.performOnPrimary( @@ -938,7 +939,7 @@ private void executeShardBulkOnReplica( globalCheckpointOnPrimary, maxSeqNoOfUpdatesOrDeletes, permitAcquiredFuture, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); final Translog.Location location; try (Releasable ignored = permitAcquiredFuture.actionGet()) { @@ -1086,7 +1087,7 @@ private void executeResyncOnReplica( globalCheckpointOnPrimary, maxSeqNoOfUpdatesOrDeletes, acquirePermitFuture, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); try (Releasable ignored = acquirePermitFuture.actionGet()) { location = TransportResyncReplicationAction.performOnReplica(request, replica); diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index f36951cec0b84..beaa800b72588 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -91,6 +91,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -137,12 +138,14 @@ public void onRecoveryFailure(RecoveryFailedException e, boolean sendShardFailur }; protected ThreadPool threadPool; + protected Executor writeExecutor; protected long primaryTerm; @Override public void setUp() throws Exception { super.setUp(); threadPool = setUpThreadPool(); + writeExecutor = threadPool.executor(ThreadPool.Names.WRITE); primaryTerm = randomIntBetween(1, 100); // use random but fixed term for creating shards failOnShardFailures(); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java index 01168e447eeb9..b7d333de80a75 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.Releasable; import org.elasticsearch.index.Index; @@ -133,7 +134,7 @@ public void onFailure(Exception e) { onFailure(e); } } - }, ThreadPool.Names.SAME); + }, EsExecutors.DIRECT_EXECUTOR_SERVICE); } @Override diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java index e37bd879ec943..0d28f99a64237 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; @@ -359,7 +360,7 @@ public void testRetryBulkShardOperations() throws Exception { followingPrimary.getLastKnownGlobalCheckpoint(), followingPrimary.getMaxSeqNoOfUpdatesOrDeletes(), permitFuture, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); try (Releasable ignored = permitFuture.get()) { TransportBulkShardOperationsAction.shardOperationOnReplica(primaryResult.replicaRequest(), replica, logger); @@ -802,7 +803,7 @@ class CcrAction extends ReplicationAction listener) { final PlainActionFuture permitFuture = new PlainActionFuture<>(); - primary.acquirePrimaryOperationPermit(permitFuture, ThreadPool.Names.SAME); + primary.acquirePrimaryOperationPermit(permitFuture, EsExecutors.DIRECT_EXECUTOR_SERVICE); final TransportWriteAction.WritePrimaryResult ccrResult; final var threadpool = mock(ThreadPool.class); final var transportService = mock(TransportService.class); @@ -838,7 +839,7 @@ protected void performOnReplica(BulkShardOperationsRequest request, IndexShard r getPrimaryShard().getLastKnownGlobalCheckpoint(), getPrimaryShard().getMaxSeqNoOfUpdatesOrDeletes(), f, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ) ) ) { diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowEngineIndexShardTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowEngineIndexShardTests.java index 2156995f2e0f6..c51b7c346272c 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowEngineIndexShardTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowEngineIndexShardTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.repositories.IndexId; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotId; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.ccr.CcrSettings; @@ -103,7 +102,7 @@ public void testDoNotFillGaps() throws Exception { releasable.close(); latch.countDown(); }); - indexShard.acquirePrimaryOperationPermit(actionListener, ThreadPool.Names.GENERIC); + indexShard.acquirePrimaryOperationPermit(actionListener, threadPool.generic()); latch.await(); assertThat(indexShard.getLocalCheckpoint(), equalTo(seqNoBeforeGap)); indexShard.refresh("test"); From 96679d28ac5af0b114219fb3ff029306ad976f01 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 21 Sep 2023 17:54:28 +0300 Subject: [PATCH 07/16] ESQL: Account for an exception being thrown when building a BytesRefArrayBlock (#99726) Account for an exception being thrown when building a BytesRefArrayBlock by registering an empty BytesRef in the BytesRefArray, rather than doing nothing --- docs/changelog/99726.yaml | 6 ++ .../gen/ConvertEvaluatorImplementer.java | 4 + .../compute/data/BytesRefBlock.java | 2 + .../compute/data/BytesRefBlockBuilder.java | 4 +- .../compute/data/X-Block.java.st | 4 + .../compute/data/X-BlockBuilder.java.st | 4 +- .../rest-api-spec/test/100_bug_fix.yml | 55 +++++++++++ .../convert/ToIPFromStringEvaluator.java | 1 + .../convert/ToStringFromBooleanEvaluator.java | 1 + .../ToStringFromDatetimeEvaluator.java | 1 + .../convert/ToStringFromDoubleEvaluator.java | 1 + .../convert/ToStringFromIPEvaluator.java | 1 + .../convert/ToStringFromIntEvaluator.java | 1 + .../convert/ToStringFromLongEvaluator.java | 1 + .../ToStringFromUnsignedLongEvaluator.java | 1 + .../convert/ToStringFromVersionEvaluator.java | 1 + .../convert/ToVersionFromStringEvaluator.java | 1 + .../function/AbstractFunctionTestCase.java | 7 +- .../expression/function/TestCaseSupplier.java | 2 +- .../function/scalar/convert/ToIPTests.java | 95 +++++++++++++++++++ 20 files changed, 183 insertions(+), 10 deletions(-) create mode 100644 docs/changelog/99726.yaml create mode 100644 x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java diff --git a/docs/changelog/99726.yaml b/docs/changelog/99726.yaml new file mode 100644 index 0000000000000..23350fdb85bd0 --- /dev/null +++ b/docs/changelog/99726.yaml @@ -0,0 +1,6 @@ +pr: 99726 +summary: "ESQL: Account for an exception being thrown when building a `BytesRefArrayBlock`" +area: ES|QL +type: bug +issues: + - 99472 diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConvertEvaluatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConvertEvaluatorImplementer.java index 256dd690a7fb6..af42d94c236f2 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConvertEvaluatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/ConvertEvaluatorImplementer.java @@ -27,6 +27,7 @@ import static org.elasticsearch.compute.gen.Types.BLOCK; import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.BYTES_REF_ARRAY; +import static org.elasticsearch.compute.gen.Types.BYTES_REF_BLOCK; import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR; import static org.elasticsearch.compute.gen.Types.SOURCE; import static org.elasticsearch.compute.gen.Types.VECTOR; @@ -166,6 +167,9 @@ private MethodSpec evalVector() { } builder.endControlFlow(); builder.addStatement("nullsMask.set(p)"); + if (resultType.equals(BYTES_REF)) { + builder.addStatement("values.append($T.NULL_VALUE)", BYTES_REF_BLOCK); + } } builder.endControlFlow(); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java index ef063ce7a80be..30cce3dbf0bad 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlock.java @@ -20,6 +20,8 @@ */ public sealed interface BytesRefBlock extends Block permits FilterBytesRefBlock, BytesRefArrayBlock, BytesRefVectorBlock { + BytesRef NULL_VALUE = new BytesRef(); + /** * Retrieves the BytesRef value stored at the given value index. * diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlockBuilder.java index aaed753cc20e1..ed80dcf28fb8e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlockBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefBlockBuilder.java @@ -17,8 +17,6 @@ */ final class BytesRefBlockBuilder extends AbstractBlockBuilder implements BytesRefBlock.Builder { - private static final BytesRef NULL_VALUE = new BytesRef(); - private BytesRefArray values; BytesRefBlockBuilder(int estimatedSize) { @@ -69,7 +67,7 @@ public BytesRefBlockBuilder endPositionEntry() { @Override protected void writeNullValue() { - values.append(NULL_VALUE); + values.append(BytesRefBlock.NULL_VALUE); } /** diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st index b66ece387f6f0..9588cdbe9c353 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st @@ -23,6 +23,10 @@ import java.io.IOException; */ public sealed interface $Type$Block extends Block permits Filter$Type$Block, $Type$ArrayBlock, $Type$VectorBlock { +$if(BytesRef)$ + BytesRef NULL_VALUE = new BytesRef(); + +$endif$ /** * Retrieves the $type$ value stored at the given value index. * diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BlockBuilder.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BlockBuilder.java.st index ff517b1c85533..fc407361c04ba 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BlockBuilder.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BlockBuilder.java.st @@ -23,8 +23,6 @@ $endif$ final class $Type$BlockBuilder extends AbstractBlockBuilder implements $Type$Block.Builder { $if(BytesRef)$ - private static final BytesRef NULL_VALUE = new BytesRef(); - private BytesRefArray values; BytesRefBlockBuilder(int estimatedSize) { @@ -96,7 +94,7 @@ $endif$ $if(BytesRef)$ @Override protected void writeNullValue() { - values.append(NULL_VALUE); + values.append(BytesRefBlock.NULL_VALUE); } $endif$ diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml new file mode 100644 index 0000000000000..21871e661e971 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml @@ -0,0 +1,55 @@ +--- +setup: + - do: + bulk: + index: test + refresh: true + body: + - { "index": { } } + - { "emp_no": 10, "ip1": "127.0", "ip2": "0.1" } + - { "index": { } } + - { "emp_no": 20 } + +--- +"Bug fix https://github.com/elastic/elasticsearch/issues/99472": + - skip: + features: warnings + - do: + warnings: + - "Line 1:37: evaluation of [to_ip(coalesce(ip1.keyword, \"255.255.255.255\"))] failed, treating result as null. Only first 20 failures recorded." + - "java.lang.IllegalArgumentException: '127.0' is not an IP string literal." + esql.query: + body: + query: 'FROM test | sort emp_no | eval ip = to_ip(coalesce(ip1.keyword, "255.255.255.255")) | keep emp_no, ip' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "ip" } + - match: { columns.1.type: "ip" } + + - length: { values: 2 } + - match: { values.0: [ 10, null ] } + - match: { values.1: [ 20, "255.255.255.255"] } + + + - do: + warnings: + - "Line 1:98: evaluation of [to_ip(x2)] failed, treating result as null. Only first 20 failures recorded." + - "java.lang.IllegalArgumentException: '127.00.1' is not an IP string literal." + esql.query: + body: + query: 'FROM test | sort emp_no | eval x1 = concat(ip1, ip2), x2 = coalesce(x1, "255.255.255.255"), x3 = to_ip(x2) | keep emp_no, x*' + + - match: { columns.0.name: "emp_no" } + - match: { columns.0.type: "long" } + - match: { columns.1.name: "x1" } + - match: { columns.1.type: "keyword" } + - match: { columns.2.name: "x2" } + - match: { columns.2.type: "keyword" } + - match: { columns.3.name: "x3" } + - match: { columns.3.type: "ip" } + + + - length: { values: 2 } + - match: { values.0: [ 10, "127.00.1", "127.00.1", null ] } + - match: { values.1: [ 20, null, "255.255.255.255", "255.255.255.255"] } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPFromStringEvaluator.java index 7befa47c6009c..76d5c58961970 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPFromStringEvaluator.java @@ -58,6 +58,7 @@ public Block evalVector(Vector v) { nullsMask = new BitSet(positionCount); } nullsMask.set(p); + values.append(BytesRefBlock.NULL_VALUE); } } return nullsMask == null diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromBooleanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromBooleanEvaluator.java index c95d64ee0f8f5..876344b1c35bc 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromBooleanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromBooleanEvaluator.java @@ -58,6 +58,7 @@ public Block evalVector(Vector v) { nullsMask = new BitSet(positionCount); } nullsMask.set(p); + values.append(BytesRefBlock.NULL_VALUE); } } return nullsMask == null diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDatetimeEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDatetimeEvaluator.java index d5f7454fc0d12..8aa5148b21de4 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDatetimeEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDatetimeEvaluator.java @@ -58,6 +58,7 @@ public Block evalVector(Vector v) { nullsMask = new BitSet(positionCount); } nullsMask.set(p); + values.append(BytesRefBlock.NULL_VALUE); } } return nullsMask == null diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDoubleEvaluator.java index deb897a4cf170..8c7994a3c0a68 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromDoubleEvaluator.java @@ -58,6 +58,7 @@ public Block evalVector(Vector v) { nullsMask = new BitSet(positionCount); } nullsMask.set(p); + values.append(BytesRefBlock.NULL_VALUE); } } return nullsMask == null diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIPEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIPEvaluator.java index dbb54eb4dbb59..4e0249939cc91 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIPEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIPEvaluator.java @@ -58,6 +58,7 @@ public Block evalVector(Vector v) { nullsMask = new BitSet(positionCount); } nullsMask.set(p); + values.append(BytesRefBlock.NULL_VALUE); } } return nullsMask == null diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIntEvaluator.java index 6c4969948dbf0..d076b38f49b91 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromIntEvaluator.java @@ -58,6 +58,7 @@ public Block evalVector(Vector v) { nullsMask = new BitSet(positionCount); } nullsMask.set(p); + values.append(BytesRefBlock.NULL_VALUE); } } return nullsMask == null diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromLongEvaluator.java index d8fd6b0ebd57d..90448cb992cd2 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromLongEvaluator.java @@ -58,6 +58,7 @@ public Block evalVector(Vector v) { nullsMask = new BitSet(positionCount); } nullsMask.set(p); + values.append(BytesRefBlock.NULL_VALUE); } } return nullsMask == null diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromUnsignedLongEvaluator.java index f1a78b375ce7e..91e31c9626b5e 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromUnsignedLongEvaluator.java @@ -58,6 +58,7 @@ public Block evalVector(Vector v) { nullsMask = new BitSet(positionCount); } nullsMask.set(p); + values.append(BytesRefBlock.NULL_VALUE); } } return nullsMask == null diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromVersionEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromVersionEvaluator.java index 7f3361587a38c..281b881bd6141 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromVersionEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromVersionEvaluator.java @@ -58,6 +58,7 @@ public Block evalVector(Vector v) { nullsMask = new BitSet(positionCount); } nullsMask.set(p); + values.append(BytesRefBlock.NULL_VALUE); } } return nullsMask == null diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionFromStringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionFromStringEvaluator.java index d8136b4ec4127..5f6b62e16de52 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionFromStringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionFromStringEvaluator.java @@ -58,6 +58,7 @@ public Block evalVector(Vector v) { nullsMask = new BitSet(positionCount); } nullsMask.set(p); + values.append(BytesRefBlock.NULL_VALUE); } } return nullsMask == null diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index d7c962ae15a20..5efd165b53173 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -327,7 +327,7 @@ protected static List anyNullIsNull(boolean entirelyNullPreser oc.evaluatorToString, oc.expectedType, nullValue(), - oc.getExpectedWarnings(), + null, oc.getExpectedTypeError() ); })); @@ -352,7 +352,7 @@ protected static List anyNullIsNull(boolean entirelyNullPreser "LiteralsEvaluator[block=null]", entirelyNullPreservesType == false && oc.getData().size() == 1 ? DataTypes.NULL : oc.expectedType, nullValue(), - oc.getExpectedWarnings(), + null, oc.getExpectedTypeError() ); })); @@ -475,7 +475,8 @@ private static String typeErrorMessage(List> validPerPosition, Lis Map.entry(Set.of(DataTypes.DOUBLE, DataTypes.NULL), "double"), Map.entry(Set.of(DataTypes.INTEGER, DataTypes.NULL), "integer"), Map.entry(Set.of(DataTypes.LONG, DataTypes.INTEGER, DataTypes.UNSIGNED_LONG, DataTypes.DOUBLE, DataTypes.NULL), "numeric"), - Map.entry(Set.of(DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.VERSION, DataTypes.NULL), "keyword, text or version") + Map.entry(Set.of(DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.VERSION, DataTypes.NULL), "keyword, text or version"), + Map.entry(Set.of(DataTypes.IP, DataTypes.KEYWORD, DataTypes.NULL), "ip or keyword") ); private static String expectedType(Set validTypes) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 073f82847e87a..626e4af81bf44 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -633,7 +633,7 @@ private static List ipCases() { ); } - private static List stringCases(DataType type) { + public static List stringCases(DataType type) { List result = new ArrayList<>(); result.add(new TypedDataSupplier("", () -> new BytesRef(""), type)); result.add( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java new file mode 100644 index 0000000000000..7f764d9de20dc --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.stringCases; +import static org.elasticsearch.xpack.ql.util.StringUtils.parseIP; +import static org.hamcrest.Matchers.equalTo; + +public class ToIPTests extends AbstractFunctionTestCase { + public ToIPTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + String read = "Attribute[channel=0]"; + String stringEvaluator = "ToIPFromStringEvaluator[field=" + read + "]"; + List suppliers = new ArrayList<>(); + + // convert from IP to IP + TestCaseSupplier.forUnaryIp(suppliers, read, DataTypes.IP, v -> v, List.of()); + + // convert any kind of string to IP, with warnings. + for (TestCaseSupplier.TypedDataSupplier supplier : stringCases(DataTypes.KEYWORD)) { + suppliers.add(new TestCaseSupplier(supplier.name(), List.of(supplier.type()), () -> { + BytesRef value = (BytesRef) supplier.supplier().get(); + TestCaseSupplier.TypedData typed = new TestCaseSupplier.TypedData(value, supplier.type(), "value"); + TestCaseSupplier.TestCase testCase = new TestCaseSupplier.TestCase( + List.of(typed), + stringEvaluator, + DataTypes.IP, + equalTo(null) + ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") + .withWarning("java.lang.IllegalArgumentException: '" + value.utf8ToString() + "' is not an IP string literal."); + return testCase; + })); + } + + // convert valid IPs shaped as strings + DataType inputType = DataTypes.KEYWORD; + for (TestCaseSupplier.TypedDataSupplier ipGen : validIPsAsStrings()) { + suppliers.add(new TestCaseSupplier(ipGen.name(), List.of(inputType), () -> { + BytesRef ip = (BytesRef) ipGen.supplier().get(); + TestCaseSupplier.TypedData typed = new TestCaseSupplier.TypedData(ip, inputType, "value"); + return new TestCaseSupplier.TestCase(List.of(typed), stringEvaluator, DataTypes.IP, equalTo(parseIP(ip.utf8ToString()))); + })); + } + + // add null as parameter + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + } + + @Override + protected Expression build(Source source, List args) { + return new ToIP(source, args.get(0)); + } + + private static List validIPsAsStrings() { + return List.of( + new TestCaseSupplier.TypedDataSupplier("<127.0.0.1 ip>", () -> new BytesRef("127.0.0.1"), DataTypes.KEYWORD), + new TestCaseSupplier.TypedDataSupplier( + "", + () -> new BytesRef(NetworkAddress.format(ESTestCase.randomIp(true))), + DataTypes.KEYWORD + ), + new TestCaseSupplier.TypedDataSupplier( + "", + () -> new BytesRef(NetworkAddress.format(ESTestCase.randomIp(false))), + DataTypes.KEYWORD + ) + ); + } +} From eb761287e14c157dfbfb45a4801a62f0afb4953b Mon Sep 17 00:00:00 2001 From: Artem Prigoda Date: Thu, 21 Sep 2023 17:13:36 +0200 Subject: [PATCH 08/16] Mute RotatableSecretTests#testBasicRotation (#99761) See https://github.com/elastic/elasticsearch/issues/99759 --- .../org/elasticsearch/common/settings/RotatableSecretTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/common/settings/RotatableSecretTests.java b/server/src/test/java/org/elasticsearch/common/settings/RotatableSecretTests.java index cf4d1c3d4f204..8fe845fb3b3db 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/RotatableSecretTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/RotatableSecretTests.java @@ -27,6 +27,7 @@ public class RotatableSecretTests extends ESTestCase { private final SecureString secret2 = new SecureString(randomAlphaOfLength(10)); private final SecureString secret3 = new SecureString(randomAlphaOfLength(10)); + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99759") public void testBasicRotation() throws Exception { // initial state RotatableSecret rotatableSecret = new RotatableSecret(secret1); From 3f32affbb693a9bd3a120506b3c3d3b23d62f6ed Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 21 Sep 2023 17:08:43 +0100 Subject: [PATCH 09/16] Add component info versions to node info in a pluggable way (#99631) This adds a `ComponentVersionNumber` service interface for modules to provide version numbers for individual components to be reported inside node info. Initial implementations for `MlConfigVersion` and `TransformConfigVersion` are provided. --- docs/changelog/99631.yaml | 5 +++ docs/reference/cluster/nodes-info.asciidoc | 13 +++++++ .../nodesinfo/SimpleNodesInfoIT.java | 20 +++++----- server/src/main/java/module-info.java | 1 + .../org/elasticsearch/TransportVersions.java | 1 + .../node/info/ComponentVersionNumber.java | 26 +++++++++++++ .../admin/cluster/node/info/NodeInfo.java | 18 +++++++++ .../cluster/node/info/NodesInfoResponse.java | 5 +++ .../org/elasticsearch/node/NodeService.java | 20 ++++++++++ .../cluster/node/info/NodeInfoTests.java | 3 ++ .../remote/RemoteClusterNodesActionTests.java | 3 ++ .../cluster/stats/ClusterStatsNodesTests.java | 1 + .../ingest/ReservedPipelineActionTests.java | 1 + .../TransportVersionsFixupListenerTests.java | 1 + .../nodesinfo/NodeInfoStreamingTests.java | 6 +++ .../action/cat/RestPluginsActionTests.java | 2 + .../AutoscalingNodesInfoServiceTests.java | 2 + .../ComponentVersionsNodesInfoIT.java | 39 +++++++++++++++++++ .../core/src/main/java/module-info.java | 5 +++ .../core/ml/MlConfigVersionComponent.java | 23 +++++++++++ .../TransformConfigVersionComponent.java | 23 +++++++++++ ...n.cluster.node.info.ComponentVersionNumber | 2 + .../TransportNodeEnrollmentActionTests.java | 1 + ...InternalEnrollmentTokenGeneratorTests.java | 3 ++ 24 files changed, 214 insertions(+), 10 deletions(-) create mode 100644 docs/changelog/99631.yaml create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/ComponentVersionNumber.java create mode 100644 x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/nodesinfo/ComponentVersionsNodesInfoIT.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersionComponent.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformConfigVersionComponent.java create mode 100644 x-pack/plugin/core/src/main/resources/META-INF/services/org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber diff --git a/docs/changelog/99631.yaml b/docs/changelog/99631.yaml new file mode 100644 index 0000000000000..d9174de76f1ea --- /dev/null +++ b/docs/changelog/99631.yaml @@ -0,0 +1,5 @@ +pr: 99631 +summary: Add component info versions to node info in a pluggable way +area: Infra/REST API +type: enhancement +issues: [] diff --git a/docs/reference/cluster/nodes-info.asciidoc b/docs/reference/cluster/nodes-info.asciidoc index 00650ce948b6f..16da5a25d1fbf 100644 --- a/docs/reference/cluster/nodes-info.asciidoc +++ b/docs/reference/cluster/nodes-info.asciidoc @@ -139,6 +139,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] `index_version`:: The most recent index version that this node can read. +`component_versions`:: + The version numbers of individual components loaded in this node. + The `os` flag can be set to retrieve information that concern the operating system: @@ -236,6 +239,10 @@ The API returns the following response: "version": "{version}", "transport_version": 100000298, "index_version": 100000074, + "component_versions": { + "ml_config_version": 100000162, + "transform_config_version": 100000096 + }, "build_flavor": "default", "build_type": "{build_type}", "build_hash": "587409e", @@ -276,6 +283,7 @@ The API returns the following response: // TESTRESPONSE[s/"ip": "192.168.17"/"ip": $body.$_path/] // TESTRESPONSE[s/"transport_version": 100000298/"transport_version": $body.$_path/] // TESTRESPONSE[s/"index_version": 100000074/"index_version": $body.$_path/] +// TESTRESPONSE[s/"component_versions": \{[^\}]*\}/"component_versions": $body.$_path/] // TESTRESPONSE[s/"build_hash": "587409e"/"build_hash": $body.$_path/] // TESTRESPONSE[s/"roles": \[[^\]]*\]/"roles": $body.$_path/] // TESTRESPONSE[s/"attributes": \{[^\}]*\}/"attributes": $body.$_path/] @@ -311,6 +319,10 @@ The API returns the following response: "version": "{version}", "transport_version": 100000298, "index_version": 100000074, + "component_versions": { + "ml_config_version": 100000162, + "transform_config_version": 100000096 + }, "build_flavor": "default", "build_type": "{build_type}", "build_hash": "587409e", @@ -375,6 +387,7 @@ The API returns the following response: // TESTRESPONSE[s/"ip": "192.168.17"/"ip": $body.$_path/] // TESTRESPONSE[s/"transport_version": 100000298/"transport_version": $body.$_path/] // TESTRESPONSE[s/"index_version": 100000074/"index_version": $body.$_path/] +// TESTRESPONSE[s/"component_versions": \{[^\}]*\}/"component_versions": $body.$_path/] // TESTRESPONSE[s/"build_hash": "587409e"/"build_hash": $body.$_path/] // TESTRESPONSE[s/"roles": \[[^\]]*\]/"roles": $body.$_path/] // TESTRESPONSE[s/"attributes": \{[^\}]*\}/"attributes": $body.$_path/] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java b/server/src/internalClusterTest/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java index 3b0829587fdf1..15225edc47a60 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java @@ -23,7 +23,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.notNullValue; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) @@ -42,29 +42,29 @@ public void testNodesInfos() throws Exception { logger.info("--> started nodes: {} and {}", server1NodeId, server2NodeId); NodesInfoResponse response = clusterAdmin().prepareNodesInfo().execute().actionGet(); - assertThat(response.getNodes().size(), is(2)); + assertThat(response.getNodes(), hasSize(2)); assertThat(response.getNodesMap().get(server1NodeId), notNullValue()); assertThat(response.getNodesMap().get(server2NodeId), notNullValue()); response = clusterAdmin().nodesInfo(new NodesInfoRequest()).actionGet(); - assertThat(response.getNodes().size(), is(2)); + assertThat(response.getNodes(), hasSize(2)); assertThat(response.getNodesMap().get(server1NodeId), notNullValue()); assertThat(response.getNodesMap().get(server2NodeId), notNullValue()); response = clusterAdmin().nodesInfo(new NodesInfoRequest(server1NodeId)).actionGet(); - assertThat(response.getNodes().size(), is(1)); + assertThat(response.getNodes(), hasSize(1)); assertThat(response.getNodesMap().get(server1NodeId), notNullValue()); response = clusterAdmin().nodesInfo(new NodesInfoRequest(server1NodeId)).actionGet(); - assertThat(response.getNodes().size(), is(1)); + assertThat(response.getNodes(), hasSize(1)); assertThat(response.getNodesMap().get(server1NodeId), notNullValue()); response = clusterAdmin().nodesInfo(new NodesInfoRequest(server2NodeId)).actionGet(); - assertThat(response.getNodes().size(), is(1)); + assertThat(response.getNodes(), hasSize(1)); assertThat(response.getNodesMap().get(server2NodeId), notNullValue()); response = clusterAdmin().nodesInfo(new NodesInfoRequest(server2NodeId)).actionGet(); - assertThat(response.getNodes().size(), is(1)); + assertThat(response.getNodes(), hasSize(1)); assertThat(response.getNodesMap().get(server2NodeId), notNullValue()); } @@ -81,7 +81,7 @@ public void testNodesInfosTotalIndexingBuffer() throws Exception { logger.info("--> started nodes: {} and {}", server1NodeId, server2NodeId); NodesInfoResponse response = clusterAdmin().prepareNodesInfo().execute().actionGet(); - assertThat(response.getNodes().size(), is(2)); + assertThat(response.getNodes(), hasSize(2)); assertThat(response.getNodesMap().get(server1NodeId), notNullValue()); assertNotNull(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer()); assertThat(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer().getBytes(), greaterThan(0L)); @@ -92,7 +92,7 @@ public void testNodesInfosTotalIndexingBuffer() throws Exception { // again, using only the indices flag response = clusterAdmin().prepareNodesInfo().clear().setIndices(true).execute().actionGet(); - assertThat(response.getNodes().size(), is(2)); + assertThat(response.getNodes(), hasSize(2)); assertThat(response.getNodesMap().get(server1NodeId), notNullValue()); assertNotNull(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer()); assertThat(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer().getBytes(), greaterThan(0L)); @@ -120,7 +120,7 @@ public void testAllocatedProcessors() throws Exception { NodesInfoResponse response = clusterAdmin().prepareNodesInfo().execute().actionGet(); - assertThat(response.getNodes().size(), is(2)); + assertThat(response.getNodes(), hasSize(2)); assertThat(response.getNodesMap().get(server1NodeId), notNullValue()); assertThat(response.getNodesMap().get(server2NodeId), notNullValue()); diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index c6e6a207ae6c0..19fe187eaa080 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -399,6 +399,7 @@ uses org.elasticsearch.internal.BuildExtension; uses org.elasticsearch.plugins.internal.SettingsExtension; uses RestExtension; + uses org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber; provides org.apache.lucene.codecs.PostingsFormat with diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 1fddeaf5d837e..926cc4801dddc 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -144,6 +144,7 @@ static TransportVersion def(int id) { public static final TransportVersion COMMIT_PRIMARY_TERM_GENERATION = def(8_501_00_1); public static final TransportVersion WAIT_FOR_CLUSTER_STATE_IN_RECOVERY_ADDED = def(8_502_00_0); public static final TransportVersion RECOVERY_COMMIT_TOO_NEW_EXCEPTION_ADDED = def(8_503_00_0); + public static final TransportVersion NODE_INFO_COMPONENT_VERSIONS_ADDED = def(8_504_00_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/ComponentVersionNumber.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/ComponentVersionNumber.java new file mode 100644 index 0000000000000..3e2572410a172 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/ComponentVersionNumber.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.node.info; + +import org.elasticsearch.common.VersionId; + +/** + * Represents a version number of a subsidiary component to be reported in node info + */ +public interface ComponentVersionNumber { + /** + * Returns the component id to report this number under. This should be in snake_case. + */ + String componentId(); + + /** + * Returns the version id to report. + */ + VersionId versionNumber(); +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java index 3086f22fae8bc..3156b9d3bac42 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java @@ -43,6 +43,7 @@ public class NodeInfo extends BaseNodeResponse { private final Version version; private final TransportVersion transportVersion; private final IndexVersion indexVersion; + private final Map componentVersions; private final Build build; @Nullable @@ -71,6 +72,11 @@ public NodeInfo(StreamInput in) throws IOException { } else { indexVersion = IndexVersion.fromId(version.id); } + if (in.getTransportVersion().onOrAfter(TransportVersions.NODE_INFO_COMPONENT_VERSIONS_ADDED)) { + componentVersions = in.readImmutableMap(StreamInput::readString, StreamInput::readVInt); + } else { + componentVersions = Map.of(); + } build = Build.readBuild(in); if (in.readBoolean()) { totalIndexingBuffer = ByteSizeValue.ofBytes(in.readLong()); @@ -102,6 +108,7 @@ public NodeInfo( Version version, TransportVersion transportVersion, IndexVersion indexVersion, + Map componentVersions, Build build, DiscoveryNode node, @Nullable Settings settings, @@ -121,6 +128,7 @@ public NodeInfo( this.version = version; this.transportVersion = transportVersion; this.indexVersion = indexVersion; + this.componentVersions = componentVersions; this.build = build; this.settings = settings; addInfoIfNonNull(OsInfo.class, os); @@ -165,6 +173,13 @@ public IndexVersion getIndexVersion() { return indexVersion; } + /** + * The version numbers of other installed components + */ + public Map getComponentVersions() { + return componentVersions; + } + /** * The build version of the node. */ @@ -219,6 +234,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.NODE_INFO_INDEX_VERSION_ADDED)) { IndexVersion.writeVersion(indexVersion, out); } + if (out.getTransportVersion().onOrAfter(TransportVersions.NODE_INFO_COMPONENT_VERSIONS_ADDED)) { + out.writeMap(componentVersions, StreamOutput::writeString, StreamOutput::writeVInt); + } Build.writeBuild(build, out); if (totalIndexingBuffer == null) { out.writeBoolean(false); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java index 68769af5a17d9..4fa99db192db5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java @@ -66,6 +66,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("version", nodeInfo.getVersion()); builder.field("transport_version", nodeInfo.getTransportVersion().id()); builder.field("index_version", nodeInfo.getIndexVersion().id()); + builder.startObject("component_versions"); + for (var cv : nodeInfo.getComponentVersions().entrySet()) { + builder.field(cv.getKey(), cv.getValue()); + } + builder.endObject(); builder.field("build_flavor", nodeInfo.getBuild().flavor()); builder.field("build_type", nodeInfo.getBuild().type().displayName()); builder.field("build_hash", nodeInfo.getBuild().hash()); diff --git a/server/src/main/java/org/elasticsearch/node/NodeService.java b/server/src/main/java/org/elasticsearch/node/NodeService.java index f71fada3c046a..7cc9b67a53959 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeService.java +++ b/server/src/main/java/org/elasticsearch/node/NodeService.java @@ -11,6 +11,7 @@ import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; @@ -19,6 +20,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.core.Assertions; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.http.HttpServerTransport; @@ -37,7 +39,10 @@ import java.io.Closeable; import java.io.IOException; +import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.stream.Collectors; public class NodeService implements Closeable { private final Settings settings; @@ -116,6 +121,7 @@ public NodeInfo info( Version.CURRENT, TransportVersion.current(), IndexVersion.current(), + findComponentVersions(), Build.current(), transportService.getLocalNode(), settings ? settingsFilter.filter(this.settings) : null, @@ -133,6 +139,20 @@ public NodeInfo info( ); } + private Map findComponentVersions() { + var versions = pluginService.loadServiceProviders(ComponentVersionNumber.class) + .stream() + .collect(Collectors.toUnmodifiableMap(ComponentVersionNumber::componentId, cvn -> cvn.versionNumber().id())); + + if (Assertions.ENABLED) { + var isSnakeCase = Pattern.compile("[a-z_]+").asMatchPredicate(); + for (String key : versions.keySet()) { + assert isSnakeCase.test(key) : "Version component " + key + " should use snake_case"; + } + } + return versions; + } + public NodeStats stats( CommonStatsFlags indices, boolean os, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java index 35f17c24b3e89..3a14bbe10a56c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java @@ -19,6 +19,8 @@ import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.index.IndexVersionUtils; +import java.util.Map; + import static java.util.Collections.emptySet; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -39,6 +41,7 @@ public void testGetInfo() { Version.CURRENT, TransportVersion.current(), IndexVersion.current(), + Map.of(), Build.current(), DiscoveryNodeUtils.builder("test_node") .roles(emptySet()) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java index 863a57a42c2bc..7c6c1ffdcf98b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -80,6 +81,7 @@ public void testDoExecuteForRemoteServerNodes() { Version.CURRENT, TransportVersion.current(), IndexVersion.current(), + Map.of(), null, node, null, @@ -150,6 +152,7 @@ public void testDoExecuteForRemoteNodes() { Version.CURRENT, TransportVersion.current(), IndexVersion.current(), + Map.of(), null, node, null, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java index 80c320493f4ac..f2b1cb28ce8b6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java @@ -325,6 +325,7 @@ private static NodeInfo createNodeInfo(String nodeId, String transportType, Stri Version.CURRENT, TransportVersion.current(), IndexVersion.current(), + Map.of(), Build.current(), DiscoveryNodeUtils.create(nodeId, buildNewFakeTransportAddress()), settings.build(), diff --git a/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java b/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java index 05b35f992564f..e02f5f66e6c80 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java @@ -103,6 +103,7 @@ public void setup() { Version.CURRENT, TransportVersion.current(), IndexVersion.current(), + Map.of(), Build.current(), discoveryNode, Settings.EMPTY, diff --git a/server/src/test/java/org/elasticsearch/cluster/service/TransportVersionsFixupListenerTests.java b/server/src/test/java/org/elasticsearch/cluster/service/TransportVersionsFixupListenerTests.java index 323c50bf23c3b..b7792c5f85207 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/TransportVersionsFixupListenerTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/TransportVersionsFixupListenerTests.java @@ -87,6 +87,7 @@ private static NodesInfoResponse getResponse(Map respo e.getValue(), null, null, + null, DiscoveryNodeUtils.create(e.getKey(), new TransportAddress(TransportAddress.META_ADDRESS, 9200)), null, null, diff --git a/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java index 8a338333065d7..c92c9c5305777 100644 --- a/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java +++ b/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java @@ -49,6 +49,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static java.util.Collections.emptySet; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; @@ -107,6 +109,9 @@ private void compareJsonOutput(ToXContent param1, ToXContent param2) throws IOEx } private static NodeInfo createNodeInfo() { + Map componentVersions = IntStream.range(0, randomInt(5)) + .boxed() + .collect(Collectors.toUnmodifiableMap(i -> randomAlphaOfLength(10), i -> randomInt(Integer.MAX_VALUE))); Build build = Build.current(); DiscoveryNode node = DiscoveryNodeUtils.builder("test_node") .roles(emptySet()) @@ -233,6 +238,7 @@ private static NodeInfo createNodeInfo() { VersionUtils.randomVersion(random()), TransportVersionUtils.randomVersion(random()), IndexVersionUtils.randomVersion(random()), + componentVersions, build, node, settings, diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestPluginsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestPluginsActionTests.java index 5249fe6aec9f0..e58127e9a6d9b 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestPluginsActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestPluginsActionTests.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import static org.hamcrest.Matchers.empty; @@ -66,6 +67,7 @@ private Table buildTable(List pluginDescriptor) { Version.CURRENT, TransportVersion.current(), IndexVersion.current(), + Map.of(), null, node(i), null, diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java index 673b05c7c2466..d9c3e813d3159 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java @@ -53,6 +53,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.RejectedExecutionException; @@ -452,6 +453,7 @@ private static org.elasticsearch.action.admin.cluster.node.info.NodeInfo infoFor Version.CURRENT, TransportVersion.current(), IndexVersion.current(), + Map.of(), Build.current(), node, null, diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/nodesinfo/ComponentVersionsNodesInfoIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/nodesinfo/ComponentVersionsNodesInfoIT.java new file mode 100644 index 0000000000000..e5131fa482b7c --- /dev/null +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/nodesinfo/ComponentVersionsNodesInfoIT.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.nodesinfo; + +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.List; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.notNullValue; + +public class ComponentVersionsNodesInfoIT extends ESIntegTestCase { + + public void testNodesInfoComponentVersions() { + List nodesIds = internalCluster().startNodes(1); + final String node_1 = nodesIds.get(0); + + ClusterHealthResponse clusterHealth = clusterAdmin().prepareHealth().setWaitForGreenStatus().setWaitForNodes("1").get(); + logger.info("--> done cluster_health, status {}", clusterHealth.getStatus()); + + String server1NodeId = internalCluster().getInstance(ClusterService.class, node_1).state().nodes().getLocalNodeId(); + logger.info("--> started nodes: {}", server1NodeId); + + NodesInfoResponse response = clusterAdmin().prepareNodesInfo().execute().actionGet(); + assertThat(response.getNodesMap().get(server1NodeId), notNullValue()); + assertThat( + response.getNodesMap().get(server1NodeId).getComponentVersions().keySet(), + containsInAnyOrder("transform_config_version", "ml_config_version") + ); + } +} diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index 3a869aca80c4f..deb3c4384a04b 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -221,4 +221,9 @@ exports org.elasticsearch.xpack.core.watcher.trigger; exports org.elasticsearch.xpack.core.watcher.watch; exports org.elasticsearch.xpack.core.watcher; + + provides org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber + with + org.elasticsearch.xpack.core.ml.MlConfigVersionComponent, + org.elasticsearch.xpack.core.transform.TransformConfigVersionComponent; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersionComponent.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersionComponent.java new file mode 100644 index 0000000000000..d100788418be8 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigVersionComponent.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml; + +import org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber; +import org.elasticsearch.common.VersionId; + +public class MlConfigVersionComponent implements ComponentVersionNumber { + @Override + public String componentId() { + return "ml_config_version"; + } + + @Override + public VersionId versionNumber() { + return MlConfigVersion.CURRENT; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformConfigVersionComponent.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformConfigVersionComponent.java new file mode 100644 index 0000000000000..e96056fa6fd7e --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformConfigVersionComponent.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.transform; + +import org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber; +import org.elasticsearch.common.VersionId; + +public class TransformConfigVersionComponent implements ComponentVersionNumber { + @Override + public String componentId() { + return "transform_config_version"; + } + + @Override + public VersionId versionNumber() { + return TransformConfigVersion.CURRENT; + } +} diff --git a/x-pack/plugin/core/src/main/resources/META-INF/services/org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber b/x-pack/plugin/core/src/main/resources/META-INF/services/org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber new file mode 100644 index 0000000000000..078217faee53a --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/META-INF/services/org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber @@ -0,0 +1,2 @@ +org.elasticsearch.xpack.core.ml.MlConfigVersionComponent +org.elasticsearch.xpack.core.transform.TransformConfigVersionComponent diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java index 6144ae74c7692..ca482decd63a4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java @@ -105,6 +105,7 @@ public void testDoExecute() throws Exception { Version.CURRENT, TransportVersion.current(), IndexVersion.current(), + Map.of(), null, n, null, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java index 02b088e245120..04bb3391f16a8 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java @@ -54,6 +54,7 @@ import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.Set; import static org.elasticsearch.test.ActionListenerUtils.anyActionListener; @@ -233,6 +234,7 @@ public Answer answerNullHttpInfo(InvocationOnMock invocationO Version.CURRENT, TransportVersion.current(), IndexVersion.current(), + Map.of(), null, DiscoveryNodeUtils.builder("1").name("node-name").roles(Set.of()).build(), null, @@ -267,6 +269,7 @@ private Answer answerWithInfo(InvocationOnMock invocationOnMo Version.CURRENT, TransportVersion.current(), IndexVersion.current(), + Map.of(), null, DiscoveryNodeUtils.builder("1").name("node-name").roles(Set.of()).build(), null, From a501a4292f4a88a08b4cb04417496083d5419eae Mon Sep 17 00:00:00 2001 From: Dianna Hohensee Date: Thu, 21 Sep 2023 15:18:22 -0400 Subject: [PATCH 10/16] Pass Executor through TransportAction constructors (#99392) Rather than passing Strings around that are eventually converted to Executors, simply pass the Executors. Add some temporary constructors for serverless compatibility. Related to #97879 --- .../DataStreamsStatsTransportAction.java | 2 +- .../GeoIpDownloaderStatsTransportAction.java | 2 +- .../action/PainlessExecuteAction.java | 2 +- .../reindex/TransportRethrottleAction.java | 2 +- .../reindex/UpdateByQueryWithScriptTests.java | 2 -- .../node/tasks/CancellableTasksIT.java | 8 ++++- ...tReplicationActionRetryOnClosedNodeIT.java | 2 +- .../ClusterFormationInfoAction.java | 2 +- .../CoordinationDiagnosticsAction.java | 2 +- .../TransportNodesHotThreadsAction.java | 2 +- .../node/info/TransportNodesInfoAction.java | 2 +- ...nsportNodesReloadSecureSettingsAction.java | 2 +- .../TransportPrevalidateShardPathAction.java | 2 +- .../node/stats/TransportNodesStatsAction.java | 2 +- .../cancel/TransportCancelTasksAction.java | 2 +- .../tasks/list/TransportListTasksAction.java | 2 +- .../node/usage/TransportNodesUsageAction.java | 2 +- .../status/TransportNodesSnapshotsStatus.java | 2 +- .../stats/TransportClusterStatsAction.java | 2 +- .../analyze/TransportAnalyzeAction.java | 2 +- .../TransportReloadAnalyzersAction.java | 2 +- .../TransportClearIndicesCacheAction.java | 2 +- ...TransportVerifyShardBeforeCloseAction.java | 2 +- .../TransportFindDanglingIndexAction.java | 2 +- .../TransportListDanglingIndicesAction.java | 2 +- .../TransportAnalyzeIndexDiskUsageAction.java | 2 +- .../flush/TransportShardFlushAction.java | 2 +- .../forcemerge/TransportForceMergeAction.java | 2 +- .../TransportGetFieldMappingsIndexAction.java | 2 +- .../TransportVerifyShardIndexBlockAction.java | 2 +- .../recovery/TransportRecoveryAction.java | 2 +- .../refresh/TransportShardRefreshAction.java | 2 +- ...ansportUnpromotableShardRefreshAction.java | 2 +- .../TransportIndicesSegmentsAction.java | 2 +- .../stats/TransportFieldUsageAction.java | 2 +- .../stats/TransportIndicesStatsAction.java | 2 +- .../query/TransportValidateQueryAction.java | 2 +- .../action/bulk/TransportBulkAction.java | 3 +- .../explain/TransportExplainAction.java | 7 ++-- .../TransportFieldCapabilitiesAction.java | 8 ++++- .../action/get/TransportGetAction.java | 13 ++++---- .../get/TransportGetFromTranslogAction.java | 2 +- .../get/TransportShardMultiGetAction.java | 20 ++++++------ ...ansportShardMultiGetFomTranslogAction.java | 2 +- .../search/TransportSearchShardsAction.java | 8 ++++- .../support/HandledTransportAction.java | 32 ++++++++++++------- .../broadcast/TransportBroadcastAction.java | 7 ++-- .../node/TransportBroadcastByNodeAction.java | 9 +++--- .../TransportBroadcastUnpromotableAction.java | 25 ++++++++++++++- .../support/nodes/TransportNodesAction.java | 26 ++++++++++++--- .../TransportReplicationAction.java | 10 +++--- .../replication/TransportWriteAction.java | 3 +- .../shard/TransportSingleShardAction.java | 9 +++--- .../support/tasks/TransportTasksAction.java | 14 +++----- .../TransportShardMultiTermsVectorAction.java | 8 +++-- .../TransportTermVectorsAction.java | 7 ++-- ...ransportNodesListGatewayStartedShards.java | 2 +- .../stats/HealthApiStatsTransportAction.java | 2 +- .../seqno/GlobalCheckpointSyncAction.java | 2 +- .../index/seqno/RetentionLeaseActions.java | 2 +- .../RetentionLeaseBackgroundSyncAction.java | 2 +- .../TransportNodesListShardStoreMetadata.java | 2 +- ...nsportDeleteDesiredBalanceActionTests.java | 1 - ...TransportGetDesiredBalanceActionTests.java | 9 ++---- .../ClusterFormationInfoActionTests.java | 9 +++--- .../MasterHistoryActionTests.java | 11 ++----- .../node/tasks/TaskManagerTestCase.java | 2 +- .../cluster/node/tasks/TestTaskPlugin.java | 4 +-- .../node/tasks/TransportTasksActionTests.java | 2 +- .../bulk/TransportBulkActionIngestTests.java | 11 ++----- .../TransportBroadcastByNodeActionTests.java | 2 +- ...sportBroadcastUnpromotableActionTests.java | 3 +- .../nodes/TransportNodesActionTests.java | 9 +++--- .../TransportReplicationActionTests.java | 3 +- ...ReplicationAllPermitsAcquisitionTests.java | 3 +- .../persistent/TestPersistentTasksPlugin.java | 2 +- .../seektracker/TransportSeekStatsAction.java | 2 +- .../action/TransportAnalyticsStatsAction.java | 2 +- .../TransportAnalyticsStatsActionTests.java | 26 ++++++++++----- .../xpack/ccr/action/ShardChangesAction.java | 2 +- .../action/TransportFollowStatsAction.java | 2 +- .../action/TransportForgetFollowerAction.java | 2 +- .../ClearCcrRestoreSessionAction.java | 8 ++++- .../GetCcrRestoreFileChunkAction.java | 8 ++++- .../PutCcrRestoreSessionAction.java | 2 +- .../TransportNodeDeprecationCheckAction.java | 2 +- .../TransportDeprecationCacheResetAction.java | 2 +- ...nsportNodeDeprecationCheckActionTests.java | 24 ++++++++++---- .../TransportDownsampleIndexerAction.java | 2 +- .../action/EnrichCoordinatorStatsAction.java | 2 +- .../action/EnrichShardMultiSearchAction.java | 2 +- .../eql/plugin/TransportEqlStatsAction.java | 2 +- .../operator/exchange/ExchangeService.java | 19 ++++------- .../esql/plugin/TransportEsqlQueryAction.java | 3 +- .../esql/plugin/TransportEsqlStatsAction.java | 2 +- .../esql/EsqlInfoTransportActionTests.java | 3 -- .../GetGlobalCheckpointsShardAction.java | 2 +- .../TransportClearDeploymentCacheAction.java | 4 +-- .../ml/action/TransportCloseJobAction.java | 3 +- .../TransportDeleteExpiredDataAction.java | 13 +++----- .../ml/action/TransportFlushJobAction.java | 4 +-- .../ml/action/TransportForecastJobAction.java | 4 +-- ...sportGetDataFrameAnalyticsStatsAction.java | 2 +- ...ransportGetDatafeedRunningStateAction.java | 2 +- .../TransportGetDeploymentStatsAction.java | 2 +- .../action/TransportGetJobsStatsAction.java | 2 +- ...portInferTrainedModelDeploymentAction.java | 4 +-- .../TransportIsolateDatafeedAction.java | 2 +- .../ml/action/TransportJobTaskAction.java | 3 +- .../ml/action/TransportKillProcessAction.java | 2 +- .../ml/action/TransportPersistJobAction.java | 4 +-- .../ml/action/TransportPostDataAction.java | 4 +-- ...TransportStopDataFrameAnalyticsAction.java | 3 +- .../action/TransportStopDatafeedAction.java | 2 +- ...sportStopTrainedModelDeploymentAction.java | 4 +-- .../TransportTrainedModelCacheInfoAction.java | 2 +- .../action/TransportUpdateProcessAction.java | 4 +-- ...TransportDeleteExpiredDataActionTests.java | 11 ++----- .../TransportMonitoringBulkActionTests.java | 9 ++---- ...rtClearRepositoriesStatsArchiveAction.java | 2 +- .../TransportRepositoriesStatsAction.java | 2 +- .../TransportDeleteRollupJobAction.java | 4 +-- .../action/TransportGetRollupCapsAction.java | 9 +++++- .../TransportGetRollupIndexCapsAction.java | 9 +++++- .../action/TransportGetRollupJobAction.java | 4 +-- .../action/TransportStartRollupAction.java | 4 +-- .../action/TransportStopRollupAction.java | 3 +- ...actTransportSearchableSnapshotsAction.java | 5 +-- ...rtClearSearchableSnapshotsCacheAction.java | 2 +- ...ansportSearchableSnapshotsStatsAction.java | 2 +- ...rtSearchableSnapshotCacheStoresAction.java | 2 +- ...rchableSnapshotsNodeCachesStatsAction.java | 2 +- .../TransportClearSecurityCacheAction.java | 2 +- .../TransportClearPrivilegesCacheAction.java | 2 +- .../realm/TransportClearRealmCacheAction.java | 2 +- .../role/TransportClearRolesCacheAction.java | 2 +- ...tServiceAccountNodesCredentialsAction.java | 2 +- .../TransportGrantApiKeyActionTests.java | 2 -- ...nsportProfileHasPrivilegesActionTests.java | 3 -- .../TransportClearRealmCacheActionTests.java | 18 ++++++++--- .../TransportCreateTokenActionTests.java | 4 --- .../TransportInvalidateTokenActionTests.java | 4 --- .../blobstore/testkit/BlobAnalyzeAction.java | 8 ++++- .../testkit/GetBlobChecksumAction.java | 8 ++++- .../testkit/RegisterAnalyzeAction.java | 8 ++++- .../testkit/RepositoryAnalyzeAction.java | 3 +- .../action/SpatialStatsTransportAction.java | 2 +- .../SpatialStatsTransportActionTests.java | 17 +++++++--- .../sql/plugin/TransportSqlStatsAction.java | 2 +- .../TransportGetTransformStatsAction.java | 4 +-- .../TransportScheduleNowTransformAction.java | 3 +- .../action/TransportStopTransformAction.java | 3 +- .../TransportUpdateTransformAction.java | 3 +- .../actions/TransportWatcherStatsAction.java | 2 +- .../TransportWatcherStatsActionTests.java | 16 ++++++++-- 155 files changed, 437 insertions(+), 318 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java index e63825509d551..029fd88abd9c6 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java @@ -70,7 +70,7 @@ public DataStreamsStatsTransportAction( actionFilters, indexNameExpressionResolver, DataStreamsStatsAction.Request::new, - ThreadPool.Names.MANAGEMENT + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT) ); this.clusterService = clusterService; this.indicesService = indicesService; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java index dcd8acd5045b9..57dea7b243d48 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java @@ -52,7 +52,7 @@ public GeoIpDownloaderStatsTransportAction( actionFilters, Request::new, NodeRequest::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.transportService = transportService; this.registry = registry; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index b098054210592..783abf5551c43 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -515,7 +515,7 @@ public TransportAction( // Creating a in-memory index is not light weight // TODO: is MANAGEMENT TP the right TP? Right now this is an admin api (see action name). Request::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.scriptService = scriptService; this.indicesServices = indicesServices; diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java index df97afa0a5053..d9c759d784075 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportRethrottleAction.java @@ -46,7 +46,7 @@ public TransportRethrottleAction( RethrottleRequest::new, ListTasksResponse::new, TaskInfo::from, - ThreadPool.Names.MANAGEMENT + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT) ); this.client = client; } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java index 059718754e815..876ddefda161b 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java @@ -52,8 +52,6 @@ protected UpdateByQueryRequest request() { @Override protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptService scriptService, UpdateByQueryRequest request) { TransportService transportService = mock(TransportService.class); - - // TODO: temporary, remove in #97879 when(transportService.getThreadPool()).thenReturn(threadPool); TransportUpdateByQueryAction transportAction = new TransportUpdateByQueryAction( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java index 2645243916d76..bbd1ea67b7ef8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java @@ -512,7 +512,13 @@ public static class TransportTestAction extends HandledTransportAction getResponseReader() { } @Override - protected String getExecutor(GetRequest request, ShardId shardId) { + protected Executor getExecutor(GetRequest request, ShardId shardId) { final ClusterState clusterState = clusterService.state(); if (clusterState.metadata().getIndexSafe(shardId.getIndex()).isSystem()) { - return executorSelector.executorForGet(shardId.getIndexName()); + return threadPool.executor(executorSelector.executorForGet(shardId.getIndexName())); } else if (indicesService.indexServiceSafe(shardId.getIndex()).getIndexSettings().isSearchThrottled()) { - return ThreadPool.Names.SEARCH_THROTTLED; + return threadPool.executor(ThreadPool.Names.SEARCH_THROTTLED); } else { return super.getExecutor(request, shardId); } @@ -168,7 +169,7 @@ protected String getExecutor(GetRequest request, ShardId shardId) { private void asyncGet(GetRequest request, ShardId shardId, ActionListener listener) throws IOException { if (request.refresh() && request.realtime() == false) { - threadPool.executor(getExecutor(request, shardId)).execute(ActionRunnable.wrap(listener, l -> { + getExecutor(request, shardId).execute(ActionRunnable.wrap(listener, l -> { var indexShard = getIndexShard(shardId); indexShard.externalRefresh("refresh_flag_get", l.map(r -> shardOperation(request, shardId))); })); @@ -218,7 +219,7 @@ private void handleGetOnUnpromotableShard(GetRequest request, IndexShard indexSh ); } } - }), TransportGetFromTranslogAction.Response::new, threadPool.executor(getExecutor(request, shardId))) + }), TransportGetFromTranslogAction.Response::new, getExecutor(request, shardId)) ); } else { // A non-real-time get with no explicit refresh requested. diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java index 37e07b3ca2e07..553da3d845b54 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetFromTranslogAction.java @@ -50,7 +50,7 @@ public class TransportGetFromTranslogAction extends HandledTransportAction< @Inject public TransportGetFromTranslogAction(TransportService transportService, IndicesService indicesService, ActionFilters actionFilters) { - super(NAME, transportService, actionFilters, Request::new, ThreadPool.Names.GET); + super(NAME, transportService, actionFilters, Request::new, transportService.getThreadPool().executor(ThreadPool.Names.GET)); this.indicesService = indicesService; } diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java index 7b7fcbc41d956..83d5e6a89d138 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java @@ -39,6 +39,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.concurrent.Executor; import static org.elasticsearch.action.get.TransportGetAction.getCurrentNodeOfPrimary; import static org.elasticsearch.core.Strings.format; @@ -72,7 +73,7 @@ public TransportShardMultiGetAction( actionFilters, indexNameExpressionResolver, MultiGetShardRequest::new, - ThreadPool.Names.GET + threadPool.executor(ThreadPool.Names.GET) ); this.indicesService = indicesService; this.executorSelector = executorSelector; @@ -144,12 +145,12 @@ protected MultiGetShardResponse shardOperation(MultiGetShardRequest request, Sha } @Override - protected String getExecutor(MultiGetShardRequest request, ShardId shardId) { + protected Executor getExecutor(MultiGetShardRequest request, ShardId shardId) { final ClusterState clusterState = clusterService.state(); if (clusterState.metadata().index(shardId.getIndex()).isSystem()) { - return executorSelector.executorForGet(shardId.getIndexName()); + return threadPool.executor(executorSelector.executorForGet(shardId.getIndexName())); } else if (indicesService.indexServiceSafe(shardId.getIndex()).getIndexSettings().isSearchThrottled()) { - return ThreadPool.Names.SEARCH_THROTTLED; + return threadPool.executor(ThreadPool.Names.SEARCH_THROTTLED); } else { return super.getExecutor(request, shardId); } @@ -205,15 +206,14 @@ private void handleMultiGetOnUnpromotableShard( indexShard.waitForSegmentGeneration( r.segmentGeneration(), listener.delegateFailureAndWrap( - (ll, aLong) -> threadPool.executor(getExecutor(request, shardId)) - .execute( - ActionRunnable.supply(ll, () -> handleLocalGets(request, r.multiGetShardResponse(), shardId)) - ) + (ll, aLong) -> getExecutor(request, shardId).execute( + ActionRunnable.supply(ll, () -> handleLocalGets(request, r.multiGetShardResponse(), shardId)) + ) ) ); } } - }), TransportShardMultiGetFomTranslogAction.Response::new, threadPool.executor(getExecutor(request, shardId))) + }), TransportShardMultiGetFomTranslogAction.Response::new, getExecutor(request, shardId)) ); } else { // A non-real-time mget with no explicit refresh requested. @@ -262,7 +262,7 @@ private void getAndAddToResponse(ShardId shardId, int location, MultiGetShardReq private void asyncShardMultiGet(MultiGetShardRequest request, ShardId shardId, ActionListener listener) throws IOException { if (request.refresh() && request.realtime() == false) { - threadPool.executor(getExecutor(request, shardId)).execute(ActionRunnable.wrap(listener, l -> { + getExecutor(request, shardId).execute(ActionRunnable.wrap(listener, l -> { var indexShard = getIndexShard(shardId); indexShard.externalRefresh("refresh_flag_mget", l.map(r -> shardOperation(request, shardId))); })); diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java index cea18fe09b6bc..032f002a675c4 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetFomTranslogAction.java @@ -48,7 +48,7 @@ protected TransportShardMultiGetFomTranslogAction( IndicesService indicesService, ActionFilters actionFilters ) { - super(NAME, transportService, actionFilters, Request::new, ThreadPool.Names.GET); + super(NAME, transportService, actionFilters, Request::new, transportService.getThreadPool().executor(ThreadPool.Names.GET)); this.indicesService = indicesService; } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java index 19af8ad8a1beb..4c8ade4d78ead 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java @@ -60,7 +60,13 @@ public TransportSearchShardsAction( SearchTransportService searchTransportService, IndexNameExpressionResolver indexNameExpressionResolver ) { - super(SearchShardsAction.NAME, transportService, actionFilters, SearchShardsRequest::new, ThreadPool.Names.SEARCH_COORDINATION); + super( + SearchShardsAction.NAME, + transportService, + actionFilters, + SearchShardsRequest::new, + transportService.getThreadPool().executor(ThreadPool.Names.SEARCH_COORDINATION) + ); this.transportService = transportService; this.transportSearchAction = transportSearchAction; this.searchService = searchService; diff --git a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java index a0e36a538913d..9255ea5daddb2 100644 --- a/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java @@ -10,12 +10,14 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportService; +import java.util.concurrent.Executor; + /** * A TransportAction that self registers a handler into the transport service */ @@ -37,11 +39,24 @@ protected HandledTransportAction( TransportService transportService, ActionFilters actionFilters, Writeable.Reader requestReader, - String executor + Executor executor ) { this(actionName, true, transportService, actionFilters, requestReader, executor); } + /** + * Temporary for serverless compatibility. TODO remove. + */ + protected HandledTransportAction( + String actionName, + TransportService transportService, + ActionFilters actionFilters, + Writeable.Reader requestReader, + String executor + ) { + this(actionName, true, transportService, actionFilters, requestReader, transportService.getThreadPool().executor(executor)); + } + protected HandledTransportAction( String actionName, boolean canTripCircuitBreaker, @@ -49,7 +64,7 @@ protected HandledTransportAction( ActionFilters actionFilters, Writeable.Reader requestReader ) { - this(actionName, canTripCircuitBreaker, transportService, actionFilters, requestReader, ThreadPool.Names.SAME); + this(actionName, canTripCircuitBreaker, transportService, actionFilters, requestReader, EsExecutors.DIRECT_EXECUTOR_SERVICE); } protected HandledTransportAction( @@ -58,17 +73,10 @@ protected HandledTransportAction( TransportService transportService, ActionFilters actionFilters, Writeable.Reader requestReader, - String executor + Executor executor ) { super(actionName, actionFilters, transportService.getTaskManager()); - transportService.registerRequestHandler( - actionName, - transportService.getThreadPool().executor(executor), - false, - canTripCircuitBreaker, - requestReader, - new TransportHandler() - ); + transportService.registerRequestHandler(actionName, executor, false, canTripCircuitBreaker, requestReader, new TransportHandler()); } class TransportHandler implements TransportRequestHandler { diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java index ddd2cc43005e2..6be22f28e8ad9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.Transports; @@ -62,15 +61,15 @@ protected TransportBroadcastAction( IndexNameExpressionResolver indexNameExpressionResolver, Writeable.Reader requestReader, Writeable.Reader shardRequestReader, - String executor + Executor executor ) { // TODO replace SAME when removing workaround for https://github.com/elastic/elasticsearch/issues/97916 - super(actionName, transportService, actionFilters, requestReader, ThreadPool.Names.SAME); + super(actionName, transportService, actionFilters, requestReader, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.clusterService = clusterService; this.transportService = transportService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.transportShardAction = actionName + "[s]"; - this.executor = transportService.getThreadPool().executor(executor); + this.executor = executor; assert this.executor != EsExecutors.DIRECT_EXECUTOR_SERVICE : "O(#shards) work must always fork to an appropriate executor"; transportService.registerRequestHandler( diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index 4f78d68daa8f2..4eeb3af4810cc 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -40,7 +40,6 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestHandler; @@ -91,7 +90,7 @@ public TransportBroadcastByNodeAction( ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Writeable.Reader request, - String executor + Executor executor ) { this(actionName, clusterService, transportService, actionFilters, indexNameExpressionResolver, request, executor, true); } @@ -103,16 +102,16 @@ public TransportBroadcastByNodeAction( ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Writeable.Reader request, - String executor, + Executor executor, boolean canTripCircuitBreaker ) { // TODO replace SAME when removing workaround for https://github.com/elastic/elasticsearch/issues/97916 - super(actionName, canTripCircuitBreaker, transportService, actionFilters, request, ThreadPool.Names.SAME); + super(actionName, canTripCircuitBreaker, transportService, actionFilters, request, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.clusterService = clusterService; this.transportService = transportService; this.indexNameExpressionResolver = indexNameExpressionResolver; - this.executor = transportService.getThreadPool().executor(executor); + this.executor = executor; assert this.executor != EsExecutors.DIRECT_EXECUTOR_SERVICE : "O(#shards) work must always fork to an appropriate executor"; transportNodeBroadcastAction = actionName + "[n]"; diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java index 579ed101b56ae..75ba30e63763b 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/unpromotable/TransportBroadcastUnpromotableAction.java @@ -44,6 +44,9 @@ public abstract class TransportBroadcastUnpromotableAction requestReader, String executor + ) { + this( + actionName, + clusterService, + transportService, + shardStateAction, + actionFilters, + requestReader, + transportService.getThreadPool().executor(executor) + ); + } + + protected TransportBroadcastUnpromotableAction( + String actionName, + ClusterService clusterService, + TransportService transportService, + ShardStateAction shardStateAction, + ActionFilters actionFilters, + Writeable.Reader requestReader, + Executor executor ) { super(actionName, transportService, actionFilters, requestReader); this.clusterService = clusterService; this.shardStateAction = shardStateAction; this.transportService = transportService; this.transportUnpromotableAction = actionName + "[u]"; - this.executor = transportService.getThreadPool().executor(executor); + this.executor = executor; transportService.registerRequestHandler( transportUnpromotableAction, diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java index b45c448ecc52c..3a4b7fb0fa3bb 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -55,6 +56,22 @@ public abstract class TransportNodesAction< private final Executor finalExecutor; + /** + * Temporary for serverless compatibility. TODO remove. + */ + protected TransportNodesAction( + String actionName, + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + Writeable.Reader request, + Writeable.Reader nodeRequest, + String executor + ) { + this(actionName, threadPool, clusterService, transportService, actionFilters, request, nodeRequest, threadPool.executor(executor)); + } + /** * @param actionName action name * @param threadPool thread-pool @@ -73,14 +90,15 @@ protected TransportNodesAction( ActionFilters actionFilters, Writeable.Reader request, Writeable.Reader nodeRequest, - String executor + Executor executor ) { // coordination can run on SAME because it's only O(#nodes) work - super(actionName, transportService, actionFilters, request, ThreadPool.Names.SAME); - assert executor.equals(ThreadPool.Names.SAME) == false : "TransportNodesAction must always fork off the transport thread"; + super(actionName, transportService, actionFilters, request, EsExecutors.DIRECT_EXECUTOR_SERVICE); + assert executor.equals(EsExecutors.DIRECT_EXECUTOR_SERVICE) == false + : "TransportNodesAction must always fork off the transport thread"; this.clusterService = Objects.requireNonNull(clusterService); this.transportService = Objects.requireNonNull(transportService); - this.finalExecutor = threadPool.executor(executor); + this.finalExecutor = executor; this.transportNodeAction = actionName + "[n]"; transportService.registerRequestHandler(transportNodeAction, finalExecutor, nodeRequest, new NodeTransportHandler()); } diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index b3f0496597009..9adf4881a1ef6 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -139,7 +139,7 @@ protected TransportReplicationAction( ActionFilters actionFilters, Writeable.Reader requestReader, Writeable.Reader replicaRequestReader, - String executor + Executor executor ) { this( settings, @@ -169,7 +169,7 @@ protected TransportReplicationAction( ActionFilters actionFilters, Writeable.Reader requestReader, Writeable.Reader replicaRequestReader, - String executor, + Executor executor, boolean syncGlobalCheckpointAfterOperation, boolean forceExecutionOnPrimary ) { @@ -179,7 +179,7 @@ protected TransportReplicationAction( this.clusterService = clusterService; this.indicesService = indicesService; this.shardStateAction = shardStateAction; - this.executor = threadPool.executor(executor); + this.executor = executor; this.transportPrimaryAction = actionName + "[p]"; this.transportReplicaAction = actionName + "[r]"; @@ -197,7 +197,7 @@ protected TransportReplicationAction( transportService.registerRequestHandler( transportPrimaryAction, - threadPool.executor(executor), + executor, forceExecutionOnPrimary, true, in -> new ConcreteShardRequest<>(requestReader, in), @@ -207,7 +207,7 @@ protected TransportReplicationAction( // we must never reject on because of thread pool capacity on replicas transportService.registerRequestHandler( transportReplicaAction, - threadPool.executor(executor), + executor, true, true, in -> new ConcreteReplicaRequest<>(replicaRequestReader, in), diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java index 6101ea234947a..ea24d7deb9aa7 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; @@ -91,7 +92,7 @@ protected TransportWriteAction( actionFilters, request, replicaRequest, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, true, forceExecutionOnPrimary ); diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java index 9d980fabc6714..f4a12cae6258d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java @@ -40,6 +40,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.concurrent.Executor; import static org.elasticsearch.action.support.TransportActions.isShardNotAvailableException; import static org.elasticsearch.core.Strings.format; @@ -58,7 +59,7 @@ public abstract class TransportSingleShardAction request, - String executor + Executor executor ) { super(actionName, actionFilters, transportService.getTaskManager()); this.threadPool = threadPool; @@ -107,7 +108,7 @@ protected void doExecute(Task task, Request request, ActionListener li protected abstract Response shardOperation(Request request, ShardId shardId) throws IOException; protected void asyncShardOperation(Request request, ShardId shardId, ActionListener listener) throws IOException { - threadPool.executor(getExecutor(request, shardId)).execute(ActionRunnable.supply(listener, () -> shardOperation(request, shardId))); + getExecutor(request, shardId).execute(ActionRunnable.supply(listener, () -> shardOperation(request, shardId))); } protected abstract Writeable.Reader getResponseReader(); @@ -284,7 +285,7 @@ public String concreteIndex() { } } - protected String getExecutor(Request request, ShardId shardId) { + protected Executor getExecutor(Request request, ShardId shardId) { return executor; } } diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index fb29e8a838db6..af87fd8cddb4b 100644 --- a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -25,10 +25,10 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportChannel; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestHandler; @@ -42,6 +42,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; /** * The base class for transport actions that are interacting with currently running tasks. @@ -68,10 +69,10 @@ protected TransportTasksAction( Writeable.Reader requestReader, Writeable.Reader responsesReader, Writeable.Reader responseReader, - String nodeExecutor + Executor nodeExecutor ) { // coordination can run on SAME because it's only O(#nodes) work - super(actionName, transportService, actionFilters, requestReader, ThreadPool.Names.SAME); + super(actionName, transportService, actionFilters, requestReader, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.clusterService = clusterService; this.transportService = transportService; this.transportNodeAction = actionName + "[n]"; @@ -79,12 +80,7 @@ protected TransportTasksAction( this.responsesReader = responsesReader; this.responseReader = responseReader; - transportService.registerRequestHandler( - transportNodeAction, - transportService.getThreadPool().executor(nodeExecutor), - NodeTaskRequest::new, - new NodeTransportHandler() - ); + transportService.registerRequestHandler(transportNodeAction, nodeExecutor, NodeTaskRequest::new, new NodeTransportHandler()); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java index 3499496097580..e7de3d7e70cfc 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java @@ -26,6 +26,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.concurrent.Executor; + import static org.elasticsearch.core.Strings.format; public class TransportShardMultiTermsVectorAction extends TransportSingleShardAction< @@ -54,7 +56,7 @@ public TransportShardMultiTermsVectorAction( actionFilters, indexNameExpressionResolver, MultiTermVectorsShardRequest::new, - ThreadPool.Names.GET + threadPool.executor(ThreadPool.Names.GET) ); this.indicesService = indicesService; } @@ -108,10 +110,10 @@ protected MultiTermVectorsShardResponse shardOperation(MultiTermVectorsShardRequ } @Override - protected String getExecutor(MultiTermVectorsShardRequest request, ShardId shardId) { + protected Executor getExecutor(MultiTermVectorsShardRequest request, ShardId shardId) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); return indexService.getIndexSettings().isSearchThrottled() - ? ThreadPool.Names.SEARCH_THROTTLED + ? threadPool.executor(ThreadPool.Names.SEARCH_THROTTLED) : super.getExecutor(request, shardId); } } diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java index 845a848637c45..1375ae7fb6c3a 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.concurrent.Executor; /** * Performs the get operation. @@ -52,7 +53,7 @@ public TransportTermVectorsAction( actionFilters, indexNameExpressionResolver, TermVectorsRequest::new, - ThreadPool.Names.GET + threadPool.executor(ThreadPool.Names.GET) ); this.indicesService = indicesService; @@ -114,10 +115,10 @@ protected Writeable.Reader getResponseReader() { } @Override - protected String getExecutor(TermVectorsRequest request, ShardId shardId) { + protected Executor getExecutor(TermVectorsRequest request, ShardId shardId) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); return indexService.getIndexSettings().isSearchThrottled() - ? ThreadPool.Names.SEARCH_THROTTLED + ? threadPool.executor(ThreadPool.Names.SEARCH_THROTTLED) : super.getExecutor(request, shardId); } } diff --git a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java index 0244e3bf77d60..a0246eb29de82 100644 --- a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java +++ b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java @@ -87,7 +87,7 @@ public TransportNodesListGatewayStartedShards( actionFilters, Request::new, NodeRequest::new, - ThreadPool.Names.FETCH_SHARD_STARTED + threadPool.executor(ThreadPool.Names.FETCH_SHARD_STARTED) ); this.settings = settings; this.nodeEnv = env; diff --git a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java index 890403a48e2e7..55d496e2b8c59 100644 --- a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java +++ b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java @@ -48,7 +48,7 @@ public HealthApiStatsTransportAction( actionFilters, HealthApiStatsAction.Request::new, HealthApiStatsAction.Request.Node::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.healthApiStats = healthApiStats; } diff --git a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java index 782a49ab7228a..1f8a863fe3e7b 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java @@ -62,7 +62,7 @@ public GlobalCheckpointSyncAction( actionFilters, Request::new, Request::new, - ThreadPool.Names.WRITE, + threadPool.executor(ThreadPool.Names.WRITE), false, true ); diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java index 2866d1b0ee13a..9cc9e79ebb11a 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java @@ -69,7 +69,7 @@ abstract static class TransportRetentionLeaseAction> extend actionFilters, indexNameExpressionResolver, requestSupplier, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.indicesService = Objects.requireNonNull(indicesService); } diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java index 06d0494b1bdec..5046ea1cb4d0d 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java @@ -81,7 +81,7 @@ public RetentionLeaseBackgroundSyncAction( actionFilters, Request::new, Request::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); } diff --git a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java index 479b42fdf2e87..bbadd7038b6e5 100644 --- a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java +++ b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java @@ -87,7 +87,7 @@ public TransportNodesListShardStoreMetadata( actionFilters, Request::new, NodeRequest::new, - ThreadPool.Names.FETCH_SHARD_STORE + threadPool.executor(ThreadPool.Names.FETCH_SHARD_STORE) ); this.settings = settings; this.indicesService = indicesService; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java index a2bfa68fdf25f..cbe4acd137b2a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java @@ -135,7 +135,6 @@ public DesiredBalance compute( var listener = new PlainActionFuture(); - // TODO: temporary, remove in #97879 TransportService transportService = mock(TransportService.class); when(transportService.getThreadPool()).thenReturn(threadPool); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java index 80f0b435645e6..1d80454fcea12 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; @@ -44,6 +43,7 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.AbstractChunkedSerializingTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.junit.Before; @@ -61,7 +61,6 @@ import static org.elasticsearch.cluster.ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -69,16 +68,12 @@ public class TransportGetDesiredBalanceActionTests extends ESAllocationTestCase private final DesiredBalanceShardsAllocator desiredBalanceShardsAllocator = mock(DesiredBalanceShardsAllocator.class); private final ClusterInfoService clusterInfoService = mock(ClusterInfoService.class); - private TransportService transportService = mock(TransportService.class); private ThreadPool threadPool = mock(ThreadPool.class); + private TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); private TransportGetDesiredBalanceAction transportGetDesiredBalanceAction; @Before public void initialize() { - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - transportGetDesiredBalanceAction = new TransportGetDesiredBalanceAction( transportService, mock(ClusterService.class), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java index f190c422e165c..38c811d367560 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/ClusterFormationInfoActionTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.monitor.StatusInfo; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -155,17 +156,15 @@ private ClusterFormationFailureHelper.ClusterFormationState getClusterFormationS } public void testTransportDoExecute() { - TransportService transportService = mock(TransportService.class); + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis()); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); ActionFilters actionFilters = mock(ActionFilters.class); ClusterService clusterService = mock(ClusterService.class); when(clusterService.getSettings()).thenReturn(Settings.EMPTY); - ThreadPool threadPool = mock(ThreadPool.class); - when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis()); Coordinator coordinator = mock(Coordinator.class); ClusterFormationFailureHelper.ClusterFormationState clusterFormationState = getClusterFormationState(); when(coordinator.getClusterFormationState()).thenReturn(clusterFormationState); - - // TODO: temporary, remove in #97879 when(transportService.getThreadPool()).thenReturn(threadPool); when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); ClusterFormationInfoAction.TransportAction action = new ClusterFormationInfoAction.TransportAction( diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java index f362327a3d8e4..327a20e9aedee 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/coordination/MasterHistoryActionTests.java @@ -16,16 +16,15 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.util.ArrayList; import java.util.List; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -76,19 +75,15 @@ private MasterHistoryAction.Response mutateMasterHistoryResponse(MasterHistoryAc } public void testTransportDoExecute() { - TransportService transportService = mock(TransportService.class); + ThreadPool threadPool = mock(ThreadPool.class); + TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); ActionFilters actionFilters = mock(ActionFilters.class); MasterHistoryService masterHistoryService = mock(MasterHistoryService.class); ClusterService clusterService = mock(ClusterService.class); when(clusterService.getSettings()).thenReturn(Settings.EMPTY); - ThreadPool threadPool = mock(ThreadPool.class); when(threadPool.relativeTimeInMillis()).thenReturn(System.currentTimeMillis()); MasterHistory masterHistory = new MasterHistory(threadPool, clusterService); when(masterHistoryService.getLocalMasterHistory()).thenReturn(masterHistory); - - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); MasterHistoryAction.TransportAction action = new MasterHistoryAction.TransportAction( transportService, actionFilters, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java index 78bf709ef42f7..6ca3e6bbe5400 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java @@ -154,7 +154,7 @@ abstract class AbstractTestNodesAction()), request, nodeRequest, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java index e8a3338559947..439c87bdeace6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java @@ -269,7 +269,7 @@ public TransportTestTaskAction(ThreadPool threadPool, ClusterService clusterServ new ActionFilters(new HashSet<>()), NodesRequest::new, NodeRequest::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); } @@ -404,7 +404,7 @@ public TransportUnblockTestTasksAction(ClusterService clusterService, TransportS UnblockTestTasksRequest::new, UnblockTestTasksResponse::new, UnblockTestTaskResponse::new, - ThreadPool.Names.MANAGEMENT + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT) ); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index b0af912519e6d..a650692eea9f4 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -265,7 +265,7 @@ protected TestTasksAction(String actionName, ClusterService clusterService, Tran TestTasksRequest::new, TestTasksResponse::new, TestTaskResponse::new, - ThreadPool.Names.MANAGEMENT + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT) ); } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 85a74174b094f..86146e41f87e1 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; @@ -45,6 +44,7 @@ import org.elasticsearch.ingest.IngestService; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool.Names; @@ -67,7 +67,6 @@ import static org.hamcrest.Matchers.sameInstance; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -179,15 +178,9 @@ class TestSingleItemBulkWriteAction extends TransportSingleItemBulkWriteAction request, Writeable.Reader nodeRequest, - String nodeExecutor + Executor nodeExecutor ) { super("indices:admin/test", threadPool, clusterService, transportService, actionFilters, request, nodeRequest, nodeExecutor); } @@ -347,7 +348,7 @@ private static class DataNodesOnlyTransportNodesAction extends TestTransportNode ActionFilters actionFilters, Writeable.Reader request, Writeable.Reader nodeRequest, - String nodeExecutor + Executor nodeExecutor ) { super(threadPool, clusterService, transportService, actionFilters, request, nodeRequest, nodeExecutor); } diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 308c8a66767fa..1c857d5554113 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Strings; @@ -1504,7 +1505,7 @@ private class TestAction extends TransportReplicationAction()), Request::new, Request::new, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, false, forceExecute ); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java index 824df41f8cfa7..e687b3b1c377f 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexService; @@ -459,7 +460,7 @@ private abstract class TestAction extends TransportReplicationAction()), Request::new, Request::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.shardId = Objects.requireNonNull(shardId); this.primary = Objects.requireNonNull(primary); diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java index 580115a1e9af3..d91d595c701eb 100644 --- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java @@ -543,7 +543,7 @@ public TransportTestTaskAction(ClusterService clusterService, TransportService t TestTasksRequest::new, TestTasksResponse::new, TestTaskResponse::new, - ThreadPool.Names.MANAGEMENT + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT) ); } diff --git a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/TransportSeekStatsAction.java b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/TransportSeekStatsAction.java index c57d05cf5c530..bfc528f04b215 100644 --- a/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/TransportSeekStatsAction.java +++ b/test/external-modules/seek-tracking-directory/src/main/java/org/elasticsearch/test/seektracker/TransportSeekStatsAction.java @@ -44,7 +44,7 @@ public TransportSeekStatsAction( actionFilters, SeekStatsRequest::new, SeekStatsRequest::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.seekStatsService = seekStatsService; } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsAction.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsAction.java index 6702e37e8dc4b..fe9b43e814d1c 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsAction.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsAction.java @@ -45,7 +45,7 @@ public TransportAnalyticsStatsAction( actionFilters, AnalyticsStatsAction.Request::new, AnalyticsStatsAction.NodeRequest::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.usage = usage; } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java index 6b0b53aedd12d..f5a081217f866 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java @@ -14,9 +14,9 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ContextParser; @@ -26,36 +26,46 @@ import org.elasticsearch.xpack.analytics.AnalyticsUsage; import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction; +import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.concurrent.TimeUnit; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class TransportAnalyticsStatsActionTests extends ESTestCase { + + private static ThreadPool threadPool; + + @Before + public void setup() { + threadPool = new TestThreadPool("TransportAnalyticsStatsActionTests"); + } + + @After + public void cleanup() { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + threadPool = null; + } + public TransportAnalyticsStatsAction action(AnalyticsUsage usage) { TransportService transportService = mock(TransportService.class); - ThreadPool threadPool = mock(ThreadPool.class); - - // TODO: temporary, remove in #97879 when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - ClusterService clusterService = mock(ClusterService.class); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("nodeId"); when(clusterService.localNode()).thenReturn(discoveryNode); ClusterName clusterName = new ClusterName("cluster_name"); when(clusterService.getClusterName()).thenReturn(clusterName); - ClusterState clusterState = mock(ClusterState.class); when(clusterState.getMetadata()).thenReturn(Metadata.EMPTY_METADATA); when(clusterService.state()).thenReturn(clusterState); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java index d31a6f681cd9e..6b990d033cae5 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java @@ -363,7 +363,7 @@ public TransportAction( actionFilters, indexNameExpressionResolver, Request::new, - ThreadPool.Names.SEARCH + threadPool.executor(ThreadPool.Names.SEARCH) ); this.indicesService = indicesService; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java index bb64dda7c241b..ecca422282736 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowStatsAction.java @@ -61,7 +61,7 @@ public TransportFollowStatsAction( FollowStatsAction.StatsRequest::new, FollowStatsAction.StatsResponses::new, FollowStatsAction.StatsResponse::new, - Ccr.CCR_THREAD_POOL_NAME + transportService.getThreadPool().executor(Ccr.CCR_THREAD_POOL_NAME) ); this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java index b7d333de80a75..b8b8fa0b32c31 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java @@ -65,7 +65,7 @@ public TransportForgetFollowerAction( Objects.requireNonNull(actionFilters), Objects.requireNonNull(indexNameExpressionResolver), ForgetFollowerAction.Request::new, - ThreadPool.Names.MANAGEMENT + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT) ); this.clusterService = clusterService; this.indicesService = Objects.requireNonNull(indicesService); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java index e13f6fd5cac80..886eda5659714 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java @@ -47,7 +47,13 @@ private TransportDeleteCcrRestoreSessionAction( TransportService transportService, CcrRestoreSourceService ccrRestoreService ) { - super(actionName, transportService, actionFilters, ClearCcrRestoreSessionRequest::new, ThreadPool.Names.GENERIC); + super( + actionName, + transportService, + actionFilters, + ClearCcrRestoreSessionRequest::new, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC) + ); TransportActionProxy.registerProxyAction(transportService, actionName, false, in -> ActionResponse.Empty.INSTANCE); this.ccrRestoreService = ccrRestoreService; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java index 283d0b51f17bf..93cd6bd32dbab 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java @@ -57,7 +57,13 @@ private TransportGetCcrRestoreFileChunkAction( ActionFilters actionFilters, CcrRestoreSourceService restoreSourceService ) { - super(actionName, transportService, actionFilters, GetCcrRestoreFileChunkRequest::new, ThreadPool.Names.GENERIC); + super( + actionName, + transportService, + actionFilters, + GetCcrRestoreFileChunkRequest::new, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC) + ); TransportActionProxy.registerProxyAction(transportService, actionName, false, GetCcrRestoreFileChunkResponse::new); this.restoreSourceService = restoreSourceService; this.bigArrays = bigArrays; diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java index 0e50d18143f2f..1354c82a715d2 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java @@ -71,7 +71,7 @@ private TransportPutCcrRestoreSessionAction( actionFilters, resolver, PutCcrRestoreSessionRequest::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.indicesService = indicesService; this.ccrRestoreService = ccrRestoreService; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java index e6c93195bf1de..84552dab42b83 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java @@ -70,7 +70,7 @@ public TransportNodeDeprecationCheckAction( actionFilters, NodesDeprecationCheckRequest::new, NodesDeprecationCheckAction.NodeRequest::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.settings = settings; this.pluginsService = pluginsService; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/TransportDeprecationCacheResetAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/TransportDeprecationCacheResetAction.java index b1ae442e97319..a73371a74b2f4 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/TransportDeprecationCacheResetAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/TransportDeprecationCacheResetAction.java @@ -50,7 +50,7 @@ public TransportDeprecationCacheResetAction( actionFilters, DeprecationCacheResetAction.Request::new, DeprecationCacheResetAction.NodeRequest::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.rateLimitingFilterForIndexing = rateLimitingFilterForIndexing; } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java index 190ac8d388bcf..80692efb7474a 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java @@ -19,26 +19,41 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.mockito.Mockito; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; public class TransportNodeDeprecationCheckActionTests extends ESTestCase { + private static ThreadPool threadPool; + + @Before + public void setup() { + threadPool = new TestThreadPool("TransportNodeDeprecationCheckActionTests"); + } + + @After + public void cleanup() { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + threadPool = null; + } + public void testNodeOperation() { Settings.Builder settingsBuilder = Settings.builder(); settingsBuilder.put("some.deprecated.property", "someValue1"); @@ -53,7 +68,6 @@ public void testNodeOperation() { settingsBuilder = Settings.builder(); settingsBuilder.put("some.bad.dynamic.property", "someValue1"); Settings dynamicSettings = settingsBuilder.build(); - ThreadPool threadPool = Mockito.mock(ThreadPool.class); final XPackLicenseState licenseState = null; Metadata metadata = Metadata.builder().transientSettings(dynamicSettings).build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); @@ -64,11 +78,7 @@ public void testNodeOperation() { DiscoveryNode node = Mockito.mock(DiscoveryNode.class); when(node.getId()).thenReturn("mock-node"); TransportService transportService = Mockito.mock(TransportService.class); - - // TODO: temporary, remove in #97879 when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - when(transportService.getLocalNode()).thenReturn(node); PluginsService pluginsService = Mockito.mock(PluginsService.class); ActionFilters actionFilters = Mockito.mock(ActionFilters.class); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java index 3fa32c9f86644..454f41c38d5c3 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java @@ -69,7 +69,7 @@ public TransportDownsampleIndexerAction( indexNameExpressionResolver, DownsampleIndexerAction.Request::new, DownsampleIndexerAction.ShardDownsampleRequest::new, - Downsample.DOWSAMPLE_TASK_THREAD_POOL_NAME + transportService.getThreadPool().executor(Downsample.DOWSAMPLE_TASK_THREAD_POOL_NAME) ); this.client = new OriginSettingClient(client, ClientHelper.ROLLUP_ORIGIN); this.clusterService = clusterService; diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java index 7b2bc64b62c98..f50995f9d5b4a 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java @@ -147,7 +147,7 @@ public TransportAction( actionFilters, Request::new, NodeRequest::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.enrichCache = enrichCache; this.coordinator = coordinator; diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 616b947b5cdd0..08cd87a78d874 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -194,7 +194,7 @@ public TransportAction( actionFilters, indexNameExpressionResolver, Request::new, - ThreadPool.Names.SEARCH + threadPool.executor(ThreadPool.Names.SEARCH) ); this.indicesService = indicesService; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlStatsAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlStatsAction.java index f54465781304a..90316d8b8db44 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlStatsAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlStatsAction.java @@ -49,7 +49,7 @@ public TransportEqlStatsAction( actionFilters, EqlStatsRequest::new, EqlStatsRequest.NodeStatsRequest::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.planExecutor = planExecutor; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java index 08e793e43c612..f6762e33bbc18 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java @@ -60,8 +60,7 @@ public final class ExchangeService extends AbstractLifecycleComponent { private static final Logger LOGGER = LogManager.getLogger(ExchangeService.class); private final ThreadPool threadPool; - private final String requestExecutorName; - private final Executor responseExecutor; + private final Executor executor; private final Map sinks = ConcurrentCollections.newConcurrentMap(); private final Map sources = ConcurrentCollections.newConcurrentMap(); @@ -70,22 +69,16 @@ public final class ExchangeService extends AbstractLifecycleComponent { public ExchangeService(Settings settings, ThreadPool threadPool, String executorName) { this.threadPool = threadPool; - this.requestExecutorName = executorName; - this.responseExecutor = threadPool.executor(executorName); + this.executor = threadPool.executor(executorName); final var inactiveInterval = settings.getAsTime(INACTIVE_SINKS_INTERVAL_SETTING, TimeValue.timeValueMinutes(5)); - this.inactiveSinksReaper = new InactiveSinksReaper(LOGGER, threadPool, this.responseExecutor, inactiveInterval); + this.inactiveSinksReaper = new InactiveSinksReaper(LOGGER, threadPool, this.executor, inactiveInterval); } public void registerTransportHandler(TransportService transportService) { - transportService.registerRequestHandler( - EXCHANGE_ACTION_NAME, - threadPool.executor(requestExecutorName), - ExchangeRequest::new, - new ExchangeTransportAction() - ); + transportService.registerRequestHandler(EXCHANGE_ACTION_NAME, this.executor, ExchangeRequest::new, new ExchangeTransportAction()); transportService.registerRequestHandler( OPEN_EXCHANGE_ACTION_NAME, - threadPool.executor(requestExecutorName), + this.executor, OpenExchangeRequest::new, new OpenExchangeRequestHandler() ); @@ -257,7 +250,7 @@ protected void runInternal() { * @param remoteNode the node where the remote exchange sink is located */ public RemoteSink newRemoteSink(Task parentTask, String exchangeId, TransportService transportService, DiscoveryNode remoteNode) { - return new TransportRemoteSink(transportService, remoteNode, parentTask, exchangeId, responseExecutor); + return new TransportRemoteSink(transportService, remoteNode, parentTask, exchangeId, executor); } record TransportRemoteSink( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index 6d859ef857bc7..fbaa812f68db7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.operator.exchange.ExchangeService; import org.elasticsearch.search.SearchService; import org.elasticsearch.tasks.CancellableTask; @@ -59,7 +60,7 @@ public TransportEsqlQueryAction( BigArrays bigArrays ) { // TODO replace SAME when removing workaround for https://github.com/elastic/elasticsearch/issues/97916 - super(EsqlQueryAction.NAME, transportService, actionFilters, EsqlQueryRequest::new, ThreadPool.Names.SAME); + super(EsqlQueryAction.NAME, transportService, actionFilters, EsqlQueryRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.planExecutor = planExecutor; this.clusterService = clusterService; this.requestExecutor = threadPool.executor(EsqlPlugin.ESQL_THREAD_POOL_NAME); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java index 356c00e52f919..cd708f3ae18ff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java @@ -52,7 +52,7 @@ public TransportEsqlStatsAction( actionFilters, EsqlStatsRequest::new, EsqlStatsRequest.NodeStatsRequest::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.planExecutor = planExecutor; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java index f0b3a89a4444b..61e218572183c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/EsqlInfoTransportActionTests.java @@ -49,11 +49,8 @@ public class EsqlInfoTransportActionTests extends ESTestCase { @Before public void init() { threadPool = new TestThreadPool(getTestName()); - - // TODO: temporary, remove in #97879 transportService = mock(TransportService.class); when(transportService.getThreadPool()).thenReturn(threadPool); - client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); } diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsShardAction.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsShardAction.java index e619f041fa866..8728c63d7df01 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsShardAction.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsShardAction.java @@ -153,7 +153,7 @@ public TransportAction( actionFilters, indexNameExpressionResolver, Request::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.indicesService = indicesService; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java index ac24036cc0b63..5ec1963ecec52 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java @@ -15,9 +15,9 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.ClearDeploymentCacheAction; import org.elasticsearch.xpack.core.ml.action.ClearDeploymentCacheAction.Request; @@ -46,7 +46,7 @@ public TransportClearDeploymentCacheAction( Request::new, Response::new, Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java index 62e09d67d95f5..178853ad4fa5b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; @@ -99,7 +100,7 @@ public TransportCloseJobAction( CloseJobAction.Request::new, CloseJobAction.Response::new, CloseJobAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.threadPool = threadPool; this.client = client; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java index 90f773a864183..f51498815c40e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java @@ -50,6 +50,7 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.concurrent.Executor; import java.util.function.BooleanSupplier; import java.util.stream.Collectors; @@ -60,7 +61,7 @@ public class TransportDeleteExpiredDataAction extends HandledTransportAction< private static final Logger logger = LogManager.getLogger(TransportDeleteExpiredDataAction.class); private final ThreadPool threadPool; - private final String executor; + private final Executor executor; private final OriginSettingClient client; private final ClusterService clusterService; private final Clock clock; @@ -81,7 +82,7 @@ public TransportDeleteExpiredDataAction( ) { this( threadPool, - MachineLearning.UTILITY_THREAD_POOL_NAME, + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME), transportService, actionFilters, client, @@ -95,7 +96,7 @@ public TransportDeleteExpiredDataAction( TransportDeleteExpiredDataAction( ThreadPool threadPool, - String executor, + Executor executor, TransportService transportService, ActionFilters actionFilters, Client client, @@ -207,11 +208,7 @@ void deleteExpiredData( // the chained calls must all run the ML utility thread pool NOT the thread // the previous action returned in which in the case of a transport_client_boss // thread is a disaster. - remover.remove( - requestsPerSecond, - new ThreadedActionListener<>(threadPool.executor(executor), nextListener), - isTimedOutSupplier - ); + remover.remove(requestsPerSecond, new ThreadedActionListener<>(executor, nextListener), isTimedOutSupplier); } else { if (haveAllPreviousDeletionsCompleted) { logger.info("Completed deletion of expired ML data"); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushJobAction.java index 5814f5a0e7922..a5fe3ad67ca06 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFlushJobAction.java @@ -10,8 +10,8 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.CancellableTask; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.FlushJobAction; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; @@ -35,7 +35,7 @@ public TransportFlushJobAction( actionFilters, FlushJobAction.Request::new, FlushJobAction.Response::new, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, processManager ); // ThreadPool.Names.SAME, because operations is executed by autodetect worker thread diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java index 792a821514a28..fa761c294a67b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportForecastJobAction.java @@ -15,9 +15,9 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.CancellableTask; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.common.notifications.AbstractAuditMessage; import org.elasticsearch.xpack.core.common.notifications.AbstractAuditor; @@ -73,7 +73,7 @@ public TransportForecastJobAction( ForecastJobAction.Request::new, ForecastJobAction.Response::new, // ThreadPool.Names.SAME, because operations is executed by autodetect worker thread - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, processManager ); this.jobResultsProvider = jobResultsProvider; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java index e9478bc2462da..0f3d27cc196ed 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java @@ -98,7 +98,7 @@ public TransportGetDataFrameAnalyticsStatsAction( GetDataFrameAnalyticsStatsAction.Request::new, GetDataFrameAnalyticsStatsAction.Response::new, in -> new QueryPage<>(in, GetDataFrameAnalyticsStatsAction.Response.Stats::new), - ThreadPool.Names.MANAGEMENT + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT) ); this.client = client; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java index 8879a6be36424..c9b85915a9fd6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedRunningStateAction.java @@ -57,7 +57,7 @@ public TransportGetDatafeedRunningStateAction( Request::new, Response::new, Response::new, - ThreadPool.Names.MANAGEMENT + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java index d5d911e4329db..36d225a943348 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDeploymentStatsAction.java @@ -66,7 +66,7 @@ public TransportGetDeploymentStatsAction( GetDeploymentStatsAction.Request::new, GetDeploymentStatsAction.Response::new, AssignmentStats::new, - ThreadPool.Names.MANAGEMENT + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java index fa63f5f9d78c1..666e6bf478429 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java @@ -84,7 +84,7 @@ public TransportGetJobsStatsAction( GetJobsStatsAction.Request::new, GetJobsStatsAction.Response::new, in -> new QueryPage<>(in, JobStats::new), - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.clusterService = clusterService; this.processManager = processManager; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java index 4c91f9e4913b3..f0f8287ab0d78 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java @@ -16,9 +16,9 @@ import org.elasticsearch.action.support.tasks.TransportTasksAction; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.CancellableTask; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.inference.results.InferenceResults; @@ -50,7 +50,7 @@ public TransportInferTrainedModelDeploymentAction( InferTrainedModelDeploymentAction.Request::new, InferTrainedModelDeploymentAction.Response::new, InferTrainedModelDeploymentAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java index 977ffbb65bd7a..4ceba52a7bd83 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java @@ -41,7 +41,7 @@ public TransportIsolateDatafeedAction(TransportService transportService, ActionF IsolateDatafeedAction.Request::new, IsolateDatafeedAction.Response::new, IsolateDatafeedAction.Response::new, - MachineLearning.UTILITY_THREAD_POOL_NAME + transportService.getThreadPool().executor(MachineLearning.UTILITY_THREAD_POOL_NAME) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java index e337a3606926b..b9e908dc724b3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.xpack.ml.job.task.JobTask; import java.util.List; +import java.util.concurrent.Executor; /** * Base class that redirects a request to a node where the job task is running. @@ -42,7 +43,7 @@ public abstract class TransportJobTaskAction requestReader, Writeable.Reader responseReader, - String nodeExecutor, + Executor nodeExecutor, AutodetectProcessManager processManager ) { super(actionName, clusterService, transportService, actionFilters, requestReader, responseReader, responseReader, nodeExecutor); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java index c64255dacd703..b79ba6a3cc46c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java @@ -58,7 +58,7 @@ public TransportKillProcessAction( KillProcessAction.Request::new, KillProcessAction.Response::new, KillProcessAction.Response::new, - MachineLearning.UTILITY_THREAD_POOL_NAME + transportService.getThreadPool().executor(MachineLearning.UTILITY_THREAD_POOL_NAME) ); this.auditor = auditor; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java index dce719fec79ac..edea864eea779 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPersistJobAction.java @@ -10,8 +10,8 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.CancellableTask; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.PersistJobAction; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; @@ -33,7 +33,7 @@ public TransportPersistJobAction( actionFilters, PersistJobAction.Request::new, PersistJobAction.Response::new, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, processManager ); // ThreadPool.Names.SAME, because operations is executed by autodetect worker thread diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java index 4867f96b03224..7df6b35d272af 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java @@ -10,9 +10,9 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.tasks.CancellableTask; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.PostDataAction; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; @@ -42,7 +42,7 @@ public TransportPostDataAction( actionFilters, PostDataAction.Request::new, PostDataAction.Response::new, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, processManager ); // ThreadPool.Names.SAME, because operations is executed by autodetect worker thread diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java index 477f5208f63de..f339376d97a03 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksService; @@ -90,7 +91,7 @@ public TransportStopDataFrameAnalyticsAction( StopDataFrameAnalyticsAction.Request::new, StopDataFrameAnalyticsAction.Response::new, StopDataFrameAnalyticsAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.threadPool = threadPool; this.persistentTasksService = persistentTasksService; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java index 632b456a17a00..614fcfaed6120 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java @@ -94,7 +94,7 @@ public TransportStopDatafeedAction( StopDatafeedAction.Request::new, StopDatafeedAction.Response::new, StopDatafeedAction.Response::new, - MachineLearning.UTILITY_THREAD_POOL_NAME + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) ); this.threadPool = Objects.requireNonNull(threadPool); this.persistentTasksService = Objects.requireNonNull(persistentTasksService); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java index c14c5d4aadf5b..6e90d097d1e9f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java @@ -24,13 +24,13 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.ingest.IngestMetadata; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.StopTrainedModelDeploymentAction; @@ -89,7 +89,7 @@ public TransportStopTrainedModelDeploymentAction( StopTrainedModelDeploymentAction.Request::new, StopTrainedModelDeploymentAction.Response::new, StopTrainedModelDeploymentAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.client = new OriginSettingClient(client, ML_ORIGIN); this.ingestService = ingestService; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java index 15fc2c69d4820..0af653e1495a3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java @@ -53,7 +53,7 @@ public TransportTrainedModelCacheInfoAction( actionFilters, TrainedModelCacheInfoAction.Request::new, NodeModelCacheInfoRequest::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.modelLoadingService = modelLoadingService; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateProcessAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateProcessAction.java index 9fb72e96514c0..80846a74dfc61 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateProcessAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateProcessAction.java @@ -10,8 +10,8 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.CancellableTask; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.UpdateProcessAction; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; @@ -34,7 +34,7 @@ public TransportUpdateProcessAction( actionFilters, UpdateProcessAction.Request::new, UpdateProcessAction.Response::new, - ThreadPool.Names.SAME, + EsExecutors.DIRECT_EXECUTOR_SERVICE, processManager ); // ThreadPool.Names.SAME, because operations is executed by autodetect worker thread diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java index 26409b1ada849..124c6c6878ac9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -38,7 +39,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; public class TransportDeleteExpiredDataActionTests extends ESTestCase { @@ -59,18 +59,13 @@ public void remove(float requestsPerSec, ActionListener listener, Boole @Before public void setup() { threadPool = new TestThreadPool("TransportDeleteExpiredDataActionTests thread pool"); - TransportService transportService = mock(TransportService.class); - - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - Client client = mock(Client.class); ClusterService clusterService = mock(ClusterService.class); auditor = mock(AnomalyDetectionAuditor.class); transportDeleteExpiredDataAction = new TransportDeleteExpiredDataAction( threadPool, - ThreadPool.Names.SAME, - transportService, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + mock(TransportService.class), new ActionFilters(Collections.emptySet()), client, clusterService, diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java index d2a694c04ae34..35a2b0c8cd58a 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java @@ -24,13 +24,13 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskAwareRequest; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockUtils; import org.elasticsearch.test.RandomObjects; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -96,13 +96,8 @@ public void setUpMocks() { exporters = mock(Exporters.class); threadPool = mock(ThreadPool.class); clusterService = mock(ClusterService.class); - transportService = mock(TransportService.class); + transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); filters = mock(ActionFilters.class); - - // TODO: temporary, remove in #97879 - when(transportService.getThreadPool()).thenReturn(threadPool); - when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); - when(transportService.getTaskManager()).thenReturn(taskManager); when(taskManager.register(anyString(), eq(MonitoringBulkAction.NAME), any(TaskAwareRequest.class))).thenReturn(mock(Task.class)); when(filters.filters()).thenReturn(new ActionFilter[0]); diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportClearRepositoriesStatsArchiveAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportClearRepositoriesStatsArchiveAction.java index b1a702c6c7a29..6872210ca303f 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportClearRepositoriesStatsArchiveAction.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportClearRepositoriesStatsArchiveAction.java @@ -49,7 +49,7 @@ public TransportClearRepositoriesStatsArchiveAction( actionFilters, ClearRepositoriesMeteringArchiveRequest::new, ClearRepositoriesStatsArchiveNodeRequest::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.repositoriesService = repositoriesService; } diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportRepositoriesStatsAction.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportRepositoriesStatsAction.java index 8c7f98592acd9..1574cce107416 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportRepositoriesStatsAction.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/TransportRepositoriesStatsAction.java @@ -47,7 +47,7 @@ public TransportRepositoriesStatsAction( actionFilters, RepositoriesMeteringRequest::new, RepositoriesNodeStatsRequest::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.repositoriesService = repositoriesService; } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java index 971a51a042b98..86477f4f8c180 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportDeleteRollupJobAction.java @@ -17,11 +17,11 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.indexing.IndexerState; @@ -47,7 +47,7 @@ public TransportDeleteRollupJobAction(TransportService transportService, ActionF DeleteRollupJobAction.Request::new, DeleteRollupJobAction.Response::new, DeleteRollupJobAction.Response::new, - ThreadPool.Names.SAME + EsExecutors.DIRECT_EXECUTOR_SERVICE ); } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java index 672bd0495afbb..7a390a789d7d4 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupCapsAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -39,7 +40,13 @@ public class TransportGetRollupCapsAction extends HandledTransportAction request, - String executor, + Executor executor, IndicesService indicesService, XPackLicenseState licenseState ) { @@ -67,7 +68,7 @@ public abstract class AbstractTransportSearchableSnapshotsAction< ActionFilters actionFilters, IndexNameExpressionResolver resolver, Writeable.Reader request, - String executor, + Executor executor, IndicesService indicesService, XPackLicenseState licenseState, boolean canTripCircuitBreaker diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java index 0fab5c4db2e46..8a4d21b4a98b8 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java @@ -43,7 +43,7 @@ public TransportClearSearchableSnapshotsCacheAction( actionFilters, indexNameExpressionResolver, ClearSearchableSnapshotsCacheRequest::new, - ThreadPool.Names.MANAGEMENT, + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT), indicesService, licenseState, false diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java index 3d518c4359b30..e4632b697f160 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java @@ -48,7 +48,7 @@ public TransportSearchableSnapshotsStatsAction( actionFilters, indexNameExpressionResolver, SearchableSnapshotsStatsRequest::new, - ThreadPool.Names.MANAGEMENT, + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT), indicesService, licenseState ); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java index 479d7eb0f3430..ce49969d3fb6c 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java @@ -62,7 +62,7 @@ public TransportSearchableSnapshotCacheStoresAction( actionFilters, Request::new, NodeRequest::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.cacheService = cacheService.get(); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java index 9a12661cc2a67..71aa7709685b1 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java @@ -72,7 +72,7 @@ public TransportSearchableSnapshotsNodeCachesStatsAction( actionFilters, NodesRequest::new, NodeRequest::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.frozenCacheService = frozenCacheService; this.licenseState = licenseState; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportClearSecurityCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportClearSecurityCacheAction.java index d33e10e6d32f3..56c8ac6a80868 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportClearSecurityCacheAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportClearSecurityCacheAction.java @@ -53,7 +53,7 @@ public TransportClearSecurityCacheAction( actionFilters, ClearSecurityCacheRequest::new, ClearSecurityCacheRequest.Node::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.cacheInvalidatorRegistry = cacheInvalidatorRegistry; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportClearPrivilegesCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportClearPrivilegesCacheAction.java index bd68ea31d9df6..bfaac8c84ec34 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportClearPrivilegesCacheAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportClearPrivilegesCacheAction.java @@ -52,7 +52,7 @@ public TransportClearPrivilegesCacheAction( actionFilters, ClearPrivilegesCacheRequest::new, ClearPrivilegesCacheRequest.Node::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.rolesStore = rolesStore; this.cacheInvalidatorRegistry = cacheInvalidatorRegistry; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java index 49d2e3e74b2c4..3daf9c4053bfa 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java @@ -54,7 +54,7 @@ public TransportClearRealmCacheAction( actionFilters, ClearRealmCacheRequest::new, ClearRealmCacheRequest.Node::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.realms = realms; this.authenticationService = authenticationService; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java index adcd1c4090b6f..7ebec117ff428 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java @@ -48,7 +48,7 @@ public TransportClearRolesCacheAction( actionFilters, ClearRolesCacheRequest::new, ClearRolesCacheRequest.Node::new, - ThreadPool.Names.MANAGEMENT + threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.rolesStore = rolesStore; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountNodesCredentialsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountNodesCredentialsAction.java index 2e41fc622cb77..8c338250f16a0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountNodesCredentialsAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountNodesCredentialsAction.java @@ -55,7 +55,7 @@ public TransportGetServiceAccountNodesCredentialsAction( actionFilters, GetServiceAccountCredentialsNodesRequest::new, GetServiceAccountCredentialsNodesRequest.Node::new, - ThreadPool.Names.GENERIC + threadPool.executor(ThreadPool.Names.GENERIC) ); this.fileServiceAccountTokenStore = fileServiceAccountTokenStore; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java index f9e219f168718..c8c996f37ebfc 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportGrantApiKeyActionTests.java @@ -82,8 +82,6 @@ public void setupMocks() throws Exception { threadPool = new TestThreadPool("TP-" + getTestName()); final ThreadContext threadContext = threadPool.getThreadContext(); - - // TODO: temporary, remove in #97879 TransportService transportService = mock(TransportService.class); when(transportService.getThreadPool()).thenReturn(threadPool); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java index dca8633d7c1eb..4155e6d4168cb 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/profile/TransportProfileHasPrivilegesActionTests.java @@ -81,10 +81,7 @@ public class TransportProfileHasPrivilegesActionTests extends ESTestCase { public void setup() { threadPool = new TestThreadPool(TransportProfileHasPrivilegesActionTests.class.getSimpleName()); transportService = mock(TransportService.class); - - // TODO: temporary, remove in #97879 when(transportService.getThreadPool()).thenReturn(threadPool); - actionFilters = mock(ActionFilters.class); authorizationService = mock(AuthorizationService.class); nativePrivilegeStore = mock(NativePrivilegeStore.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java index 92bef206a9d6a..8a1d6bfc069d4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheActionTests.java @@ -14,7 +14,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; +import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest; @@ -24,11 +24,13 @@ import org.elasticsearch.xpack.core.security.authc.support.CachingRealm; import org.elasticsearch.xpack.security.authc.AuthenticationService; import org.elasticsearch.xpack.security.authc.Realms; +import org.junit.After; import org.junit.Before; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.ArgumentMatchers.any; @@ -39,6 +41,7 @@ public class TransportClearRealmCacheActionTests extends ESTestCase { + private static ThreadPool threadPool; private AuthenticationService authenticationService; private TransportClearRealmCacheAction action; private TestCachingRealm nativeRealm; @@ -46,13 +49,14 @@ public class TransportClearRealmCacheActionTests extends ESTestCase { @Before public void setup() { + threadPool = new TestThreadPool("TransportClearRealmCacheActionTests"); + authenticationService = mock(AuthenticationService.class); nativeRealm = mockRealm("native"); fileRealm = mockRealm("file"); final Realms realms = mockRealms(List.of(nativeRealm, fileRealm)); - - ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); + TransportService transportService = mock(TransportService.class); + when(transportService.getThreadPool()).thenReturn(threadPool); action = new TransportClearRealmCacheAction( threadPool, @@ -64,6 +68,12 @@ public void setup() { ); } + @After + public void cleanup() { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + threadPool = null; + } + public void testSingleUserCacheCleanupForAllRealms() { final String user = "test"; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java index dcdeace6b06b6..fe48408b39c54 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java @@ -100,11 +100,7 @@ public class TransportCreateTokenActionTests extends ESTestCase { @Before public void setupClient() { threadPool = new TestThreadPool(getTestName()); - - // TODO: temporary, remove in #97879 transportService = mock(TransportService.class); - when(transportService.getThreadPool()).thenReturn(threadPool); - client = mock(Client.class); idxReqReference = new AtomicReference<>(); authenticationService = mock(AuthenticationService.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java index f07bb7dc365ae..a0f7892c3319d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java @@ -66,11 +66,7 @@ public class TransportInvalidateTokenActionTests extends ESTestCase { @Before public void setup() { threadPool = new TestThreadPool(getTestName()); - - // TODO: temporary, remove in #97879 transportService = mock(TransportService.class); - when(transportService.getThreadPool()).thenReturn(threadPool); - securityContext = new SecurityContext(Settings.EMPTY, threadPool.getThreadContext()); client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java index e5ebe47d177ff..5c9e0fc0b9202 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java @@ -167,7 +167,13 @@ public static class TransportAction extends HandledTransportAction Date: Thu, 21 Sep 2023 15:47:05 -0400 Subject: [PATCH 11/16] [ML] Adding retries for starting model deployment (#99673) * Reducing chunk size and adding retries * Testing search part * Update docs/changelog/99673.yaml * Unwrapping exception * Updating changelog * Wrapping exception and tests * Adding comments --- docs/changelog/99673.yaml | 5 + .../TrainedModelAssignmentNodeService.java | 8 + .../ChunkedTrainedModelRestorer.java | 57 +++++- .../ChunkedTrainedModelRestorerTests.java | 171 ++++++++++++++++++ 4 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/99673.yaml create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorerTests.java diff --git a/docs/changelog/99673.yaml b/docs/changelog/99673.yaml new file mode 100644 index 0000000000000..b48d620b21f49 --- /dev/null +++ b/docs/changelog/99673.yaml @@ -0,0 +1,5 @@ +pr: 99673 +summary: Adding retry logic for start model deployment API +area: Machine Learning +type: bug +issues: [ ] diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java index f1b9298b05ca4..e31f8d9992b64 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java @@ -217,8 +217,16 @@ void loadQueuedModels() { logger.debug(() -> "[" + deploymentId + "] Start deployment failed as model [" + modelId + "] was not found", ex); handleLoadFailure(loadingTask, ExceptionsHelper.missingTrainedModel(modelId, ex)); } else if (ExceptionsHelper.unwrapCause(ex) instanceof SearchPhaseExecutionException) { + /* + * This case will not catch the ElasticsearchException generated from the ChunkedTrainedModelRestorer in a scenario + * where the maximum number of retries for a SearchPhaseExecutionException or CBE occur. This is intentional. If the + * retry logic fails after retrying we should return the error and not retry here. The generated + * ElasticsearchException will contain the SearchPhaseExecutionException or CBE but cannot be unwrapped. + */ logger.debug(() -> "[" + deploymentId + "] Start deployment failed, will retry", ex); // A search phase execution failure should be retried, push task back to the queue + + // This will cause the entire model to be reloaded (all the chunks) loadingToRetry.add(loadingTask); } else { handleLoadFailure(loadingTask, ex); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java index 2ba81386fa754..015b88552a1d0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorer.java @@ -9,16 +9,20 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.sort.SortBuilders; @@ -36,6 +40,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import static org.elasticsearch.core.Strings.format; @@ -58,6 +63,8 @@ public class ChunkedTrainedModelRestorer { private static final Logger logger = LogManager.getLogger(ChunkedTrainedModelRestorer.class); private static final int MAX_NUM_DEFINITION_DOCS = 20; + private static final int SEARCH_RETRY_LIMIT = 5; + private static final TimeValue SEARCH_FAILURE_RETRY_WAIT_TIME = new TimeValue(5, TimeUnit.SECONDS); private final Client client; private final NamedXContentRegistry xContentRegistry; @@ -142,7 +149,14 @@ private void doSearch( UTILITY_THREAD_POOL_NAME, Thread.currentThread().getName() ); - SearchResponse searchResponse = client.search(searchRequest).actionGet(); + + SearchResponse searchResponse = retryingSearch( + client, + modelId, + searchRequest, + SEARCH_RETRY_LIMIT, + SEARCH_FAILURE_RETRY_WAIT_TIME + ); if (searchResponse.getHits().getHits().length == 0) { errorConsumer.accept(new ResourceNotFoundException(Messages.getMessage(Messages.MODEL_DEFINITION_NOT_FOUND, modelId))); return; @@ -201,6 +215,47 @@ private void doSearch( } } + static SearchResponse retryingSearch(Client client, String modelId, SearchRequest searchRequest, int retries, TimeValue sleep) + throws InterruptedException { + int failureCount = 0; + + while (true) { + try { + return client.search(searchRequest).actionGet(); + } catch (Exception e) { + if (ExceptionsHelper.unwrapCause(e) instanceof SearchPhaseExecutionException == false + && ExceptionsHelper.unwrapCause(e) instanceof CircuitBreakingException == false) { + throw e; + } + + if (failureCount >= retries) { + logger.warn(format("[%s] searching for model part failed %s times, returning failure", modelId, retries)); + /* + * ElasticsearchException does not implement the ElasticsearchWrapperException interface so this exception cannot + * be unwrapped. This is important because the TrainedModelAssignmentNodeService has retry logic when a + * SearchPhaseExecutionException occurs: + * https://github.com/elastic/elasticsearch/blob/main/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java#L219 + * This intentionally prevents that code from attempting to retry loading the entire model. If the retry logic here + * fails after the set retries we should not retry loading the entire model to avoid additional strain on the cluster. + */ + throw new ElasticsearchException( + format( + "loading model [%s] failed after [%s] retries. The deployment is now in a failed state, " + + "the error may be transient please stop the deployment and restart", + modelId, + retries + ), + e + ); + } + + failureCount++; + logger.debug(format("[%s] searching for model part failed %s times, retrying", modelId, failureCount)); + TimeUnit.SECONDS.sleep(sleep.getSeconds()); + } + } + } + private static SearchRequestBuilder buildSearchBuilder(Client client, String modelId, String index, int searchSize) { return client.prepareSearch(index) .setQuery( diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorerTests.java new file mode 100644 index 0000000000000..abdd1def956f0 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/persistence/ChunkedTrainedModelRestorerTests.java @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.inference.persistence; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ChunkedTrainedModelRestorerTests extends ESTestCase { + public void testRetryingSearch_ReturnsSearchResults() throws InterruptedException { + var mockClient = mock(Client.class); + var mockSearchResponse = mock(SearchResponse.class, RETURNS_DEEP_STUBS); + + PlainActionFuture searchFuture = new PlainActionFuture<>(); + searchFuture.onResponse(mockSearchResponse); + when(mockClient.search(any())).thenReturn(searchFuture); + + var request = createSearchRequest(); + + assertThat( + ChunkedTrainedModelRestorer.retryingSearch(mockClient, "", request, 5, new TimeValue(1, TimeUnit.NANOSECONDS)), + is(mockSearchResponse) + ); + + verify(mockClient, times(1)).search(any()); + } + + public void testRetryingSearch_ThrowsSearchPhaseExceptionWithNoRetries() { + try (var mockClient = mock(Client.class)) { + var searchPhaseException = new SearchPhaseExecutionException("phase", "error", ShardSearchFailure.EMPTY_ARRAY); + when(mockClient.search(any())).thenThrow(searchPhaseException); + + var request = createSearchRequest(); + + ElasticsearchException exception = expectThrows( + ElasticsearchException.class, + () -> ChunkedTrainedModelRestorer.retryingSearch(mockClient, "1", request, 0, new TimeValue(1, TimeUnit.NANOSECONDS)) + ); + + assertThat(exception.getCause(), is(searchPhaseException)); + assertThat( + exception.getMessage(), + is( + "loading model [1] failed after [0] retries. The deployment is now in a failed state, the error may be " + + "transient please stop the deployment and restart" + ) + ); + verify(mockClient, times(1)).search(any()); + } + } + + public void testRetryingSearch_ThrowsSearchPhaseExceptionAfterOneRetry() { + try (var mockClient = mock(Client.class)) { + var searchPhaseException = new SearchPhaseExecutionException("phase", "error", ShardSearchFailure.EMPTY_ARRAY); + when(mockClient.search(any())).thenThrow(searchPhaseException); + + var request = createSearchRequest(); + + ElasticsearchException exception = expectThrows( + ElasticsearchException.class, + () -> ChunkedTrainedModelRestorer.retryingSearch(mockClient, "", request, 1, new TimeValue(1, TimeUnit.NANOSECONDS)) + ); + + assertThat(exception.getCause(), is(searchPhaseException)); + verify(mockClient, times(2)).search(any()); + } + } + + public void testRetryingSearch_ThrowsCircuitBreakingExceptionAfterOneRetry_FromSearchPhaseException() { + try (var mockClient = mock(Client.class)) { + var searchPhaseException = new SearchPhaseExecutionException("phase", "error", ShardSearchFailure.EMPTY_ARRAY); + var circuitBreakerException = new CircuitBreakingException("error", CircuitBreaker.Durability.TRANSIENT); + when(mockClient.search(any())).thenThrow(searchPhaseException).thenThrow(circuitBreakerException); + + var request = createSearchRequest(); + + ElasticsearchException exception = expectThrows( + ElasticsearchException.class, + () -> ChunkedTrainedModelRestorer.retryingSearch(mockClient, "", request, 1, new TimeValue(1, TimeUnit.NANOSECONDS)) + ); + + assertThat(exception.getCause(), is(circuitBreakerException)); + verify(mockClient, times(2)).search(any()); + } + } + + public void testRetryingSearch_EnsureExceptionCannotBeUnwrapped() { + try (var mockClient = mock(Client.class)) { + var searchPhaseExecutionException = new SearchPhaseExecutionException("phase", "error", ShardSearchFailure.EMPTY_ARRAY); + when(mockClient.search(any())).thenThrow(searchPhaseExecutionException); + + var request = createSearchRequest(); + + ElasticsearchException exception = expectThrows( + ElasticsearchException.class, + () -> ChunkedTrainedModelRestorer.retryingSearch(mockClient, "", request, 1, new TimeValue(1, TimeUnit.NANOSECONDS)) + ); + + assertThat(ExceptionsHelper.unwrapCause(exception), is(exception)); + assertThat(ExceptionsHelper.unwrapCause(exception), instanceOf(ElasticsearchException.class)); + verify(mockClient, times(2)).search(any()); + } + } + + public void testRetryingSearch_ThrowsIllegalArgumentExceptionIgnoringRetries() { + try (var mockClient = mock(Client.class)) { + var exception = new IllegalArgumentException("Error"); + when(mockClient.search(any())).thenThrow(exception); + + var request = createSearchRequest(); + + IllegalArgumentException thrownException = expectThrows( + IllegalArgumentException.class, + () -> ChunkedTrainedModelRestorer.retryingSearch(mockClient, "", request, 1, new TimeValue(1, TimeUnit.NANOSECONDS)) + ); + + assertThat(thrownException, is(exception)); + verify(mockClient, times(1)).search(any()); + } + } + + public void testRetryingSearch_ThrowsSearchPhaseExceptionOnce_ThenReturnsResponse() throws InterruptedException { + try (var mockClient = mock(Client.class)) { + var mockSearchResponse = mock(SearchResponse.class, RETURNS_DEEP_STUBS); + + PlainActionFuture searchFuture = new PlainActionFuture<>(); + searchFuture.onResponse(mockSearchResponse); + + var searchPhaseException = new SearchPhaseExecutionException("phase", "error", ShardSearchFailure.EMPTY_ARRAY); + when(mockClient.search(any())).thenThrow(searchPhaseException).thenReturn(searchFuture); + + var request = createSearchRequest(); + + assertThat( + ChunkedTrainedModelRestorer.retryingSearch(mockClient, "", request, 1, new TimeValue(1, TimeUnit.NANOSECONDS)), + is(mockSearchResponse) + ); + + verify(mockClient, times(2)).search(any()); + } + } + + private static SearchRequest createSearchRequest() { + return new SearchRequest("index"); + } +} From fcf107a044ece838b7cfc552aa3dd1f108d7da5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Thu, 21 Sep 2023 22:26:20 +0200 Subject: [PATCH 12/16] [Transform] Ignore "index not found" error when `delete_dest_index` flag is set but the dest index doesn't exist (#99738) --- docs/changelog/99738.yaml | 6 ++++++ .../integration/TransformDeleteIT.java | 21 ++++++++++++++++--- .../integration/TransformRestTestCase.java | 9 +++++--- .../TransportDeleteTransformAction.java | 12 ++++++++++- 4 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 docs/changelog/99738.yaml diff --git a/docs/changelog/99738.yaml b/docs/changelog/99738.yaml new file mode 100644 index 0000000000000..1b65926aed741 --- /dev/null +++ b/docs/changelog/99738.yaml @@ -0,0 +1,6 @@ +pr: 99738 +summary: Ignore "index not found" error when `delete_dest_index` flag is set but the + dest index doesn't exist +area: Transform +type: bug +issues: [] diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java index 31e8ddd97db06..bb68c7b84da5d 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeleteIT.java @@ -108,7 +108,7 @@ public void testDeleteWithParamDeletesAutoCreatedDestinationIndex() throws Excep assertTrue(indexExists(transformDest)); assertTrue(aliasExists(transformDestAlias)); - deleteTransform(transformId, true); + deleteTransform(transformId, false, true); assertFalse(indexExists(transformDest)); assertFalse(aliasExists(transformDest)); } @@ -134,7 +134,7 @@ public void testDeleteWithParamDeletesManuallyCreatedDestinationIndex() throws E assertTrue(indexExists(transformDest)); assertTrue(aliasExists(transformDestAlias)); - deleteTransform(transformId, true); + deleteTransform(transformId, false, true); assertFalse(indexExists(transformDest)); assertFalse(aliasExists(transformDestAlias)); } @@ -158,7 +158,7 @@ public void testDeleteWithParamDoesNotDeleteManuallySetUpAlias() throws Exceptio assertTrue(indexExists(transformDest)); assertTrue(aliasExists(transformDestAlias)); - ResponseException e = expectThrows(ResponseException.class, () -> deleteTransform(transformId, true)); + ResponseException e = expectThrows(ResponseException.class, () -> deleteTransform(transformId, false, true)); assertThat( e.getMessage(), containsString( @@ -170,6 +170,21 @@ public void testDeleteWithParamDoesNotDeleteManuallySetUpAlias() throws Exceptio ); } + public void testDeleteDestinationIndexIsNoOpWhenNoDestinationIndexExists() throws Exception { + String transformId = "transform-5"; + String transformDest = transformId + "_idx"; + String transformDestAlias = transformId + "_alias"; + setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest, transformDestAlias); + + createTransform(transformId, transformDest, transformDestAlias); + assertFalse(indexExists(transformDest)); + assertFalse(aliasExists(transformDestAlias)); + + deleteTransform(transformId, false, true); + assertFalse(indexExists(transformDest)); + assertFalse(aliasExists(transformDestAlias)); + } + private void createTransform(String transformId, String destIndex, String destAlias) throws IOException { final Request createTransformRequest = createRequestWithAuth( "PUT", diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java index 77f7ecb273483..e6388bb6fea5d 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java @@ -611,12 +611,15 @@ protected static Map getTransformsStateAndStats(int from, int si } protected static void deleteTransform(String transformId) throws IOException { - deleteTransform(transformId, false); + // Ignore 404s because they imply someone was racing us to delete this transform. + deleteTransform(transformId, true, false); } - protected static void deleteTransform(String transformId, boolean deleteDestIndex) throws IOException { + protected static void deleteTransform(String transformId, boolean ignoreNotFound, boolean deleteDestIndex) throws IOException { Request request = new Request("DELETE", getTransformEndpoint() + transformId); - request.addParameter("ignore", "404"); // Ignore 404s because they imply someone was racing us to delete this + if (ignoreNotFound) { + request.addParameter("ignore", "404"); + } if (deleteDestIndex) { request.addParameter(TransformField.DELETE_DEST_INDEX.getPreferredName(), Boolean.TRUE.toString()); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java index 52a2bfc011194..9e94fbb3e5de9 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportDeleteTransformAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -135,6 +136,15 @@ private void deleteDestinationIndex( TimeValue timeout, ActionListener listener ) { + // <3> Check if the error is "index not found" error. If so, just move on. The index is already deleted. + ActionListener deleteDestIndexListener = ActionListener.wrap(listener::onResponse, e -> { + if (e instanceof IndexNotFoundException) { + listener.onResponse(AcknowledgedResponse.TRUE); + } else { + listener.onFailure(e); + } + }); + // <2> Delete destination index ActionListener> getTransformConfigurationListener = ActionListener.wrap( transformConfigAndVersion -> { @@ -149,7 +159,7 @@ private void deleteDestinationIndex( client, DeleteIndexAction.INSTANCE, deleteDestIndexRequest, - listener + deleteDestIndexListener ); }, listener::onFailure From 571cebc4155d4e20fdc3889b782258d9826fd8ef Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 21 Sep 2023 23:49:04 -0400 Subject: [PATCH 13/16] Warn about third-party JVM agents in docs (#99718) * Warn about third-party JVM agents in docs * Reword, and move warning to its own section Co-authored-by: James Rodewig --- docs/reference/setup/install.asciidoc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc index 9d47e0e80bb47..858902bb72ef2 100644 --- a/docs/reference/setup/install.asciidoc +++ b/docs/reference/setup/install.asciidoc @@ -90,6 +90,14 @@ the bundled JVM are treated as if they were within {es} itself. The bundled JVM is located within the `jdk` subdirectory of the {es} home directory. You may remove this directory if using your own JVM. +[discrete] +[[jvm-agents]] +=== JVM and Java agents + +Don't use third-party Java agents that attach to the JVM. These agents +can reduce {es} performance, including freezing or crashing nodes. + + include::install/targz.asciidoc[] include::install/zip-windows.asciidoc[] From dc8d84d2d53660923a27afea33d6f0779ccdc634 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Fri, 22 Sep 2023 10:51:24 +0300 Subject: [PATCH 14/16] Apply mixed tests after Data Stream Lifecycle went GA (#99795) --- .../test/indices.get_index_template/10_basic.yml | 12 ++++++------ .../indices.simulate_index_template/10_basic.yml | 4 ++-- .../test/indices.simulate_template/10_basic.yml | 4 ++-- .../resources/rest-api-spec/test/dlm/10_usage.yml | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml index 2c4339d8c9b9a..41e5506c412cd 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml @@ -107,8 +107,8 @@ setup: --- "Add data stream lifecycle": - skip: - version: " - 8.9.99" - reason: "Data stream lifecycle in index templates was updated after 8.9" + version: " - 8.10.99" + reason: "Data stream lifecycle in index templates was updated after 8.10" features: allowed_warnings - do: @@ -142,8 +142,8 @@ setup: --- "Get data stream lifecycle with default rollover": - skip: - version: " - 8.9.99" - reason: "Data stream lifecycle in index templates was updated after 8.9" + version: " - 8.10.99" + reason: "Data stream lifecycle in index templates was updated after 8.10" features: allowed_warnings - do: @@ -171,8 +171,8 @@ setup: --- "Reject data stream lifecycle without data stream configuration": - skip: - version: " - 8.9.99" - reason: "Data stream lifecycle in index templates was updated after 8.9" + version: " - 8.10.99" + reason: "Data stream lifecycle in index templates was updated after 8.10" - do: catch: bad_request indices.put_index_template: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml index fffb7e10e68cb..7256c1736ebd9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml @@ -227,8 +227,8 @@ --- "Simulate index template with lifecycle and include defaults": - skip: - version: " - 8.9.99" - reason: "Lifecycle is only available in 8.10+" + version: " - 8.10.99" + reason: "Lifecycle is only available in 8.11+" features: ["default_shards"] - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml index b5b1de1bcd106..887a2bc2ce705 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml @@ -202,8 +202,8 @@ --- "Simulate template with lifecycle and include defaults": - skip: - version: " - 8.9.99" - reason: "Lifecycle is only available in 8.10+" + version: " - 8.10.99" + reason: "Lifecycle is only available in 8.11+" features: ["default_shards"] - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/dlm/10_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/dlm/10_usage.yml index 8524f39b5d2cd..3033c83af8e33 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/dlm/10_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/dlm/10_usage.yml @@ -1,8 +1,8 @@ --- "Test data stream lifecycle usage stats": - skip: - version: "- 8.9.99" - reason: "the data stream lifecycle stats were updated to the usage api in 8.10" + version: "- 8.10.99" + reason: "the data stream lifecycle stats were updated to the usage api in 8.11" features: allowed_warnings - do: From 2df6def54fbff76a1d0108cbef7d9dca46757741 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 22 Sep 2023 11:20:16 +0200 Subject: [PATCH 15/16] Dry up + cleanup bits of dynamic mapping parsing (#99697) Just a few obvious dry-up spots and some dead code removal. --- .../index/mapper/DocumentParser.java | 87 +++++++++---------- .../index/mapper/NestedObjectMapper.java | 18 +--- .../index/mapper/ObjectMapper.java | 10 ++- 3 files changed, 51 insertions(+), 64 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 03653322c383c..616c48a536eef 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -398,8 +398,7 @@ private static void addFields(IndexVersion indexCreatedVersion, LuceneDocument n static void parseObjectOrField(DocumentParserContext context, Mapper mapper) throws IOException { if (mapper instanceof ObjectMapper objectMapper) { - context = context.createChildContext(objectMapper); - parseObjectOrNested(context); + parseObjectOrNested(context.createChildContext(objectMapper)); } else if (mapper instanceof FieldMapper fieldMapper) { if (shouldFlattenObject(context, fieldMapper)) { // we pass the mapper's simpleName as parentName to the new DocumentParserContext @@ -462,24 +461,25 @@ private static void parseObject(final DocumentParserContext context, String curr assert currentFieldName != null; Mapper objectMapper = context.getMapper(currentFieldName); if (objectMapper != null) { - context.path().add(currentFieldName); - if (objectMapper instanceof ObjectMapper objMapper) { - if (objMapper.subobjects() == false) { - context.path().setWithinLeafObject(true); - } - } - parseObjectOrField(context, objectMapper); - context.path().setWithinLeafObject(false); - context.path().remove(); + doParseObject(context, currentFieldName, objectMapper); } else { parseObjectDynamic(context, currentFieldName); } } + private static void doParseObject(DocumentParserContext context, String currentFieldName, Mapper objectMapper) throws IOException { + context.path().add(currentFieldName); + if (objectMapper instanceof ObjectMapper objMapper && objMapper.subobjects() == false) { + context.path().setWithinLeafObject(true); + } + parseObjectOrField(context, objectMapper); + context.path().setWithinLeafObject(false); + context.path().remove(); + } + private static void parseObjectDynamic(DocumentParserContext context, String currentFieldName) throws IOException { - if (context.dynamic() == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(context.parser().getTokenLocation(), context.parent().fullPath(), currentFieldName); - } else if (context.dynamic() == ObjectMapper.Dynamic.FALSE) { + ensureNotStrict(context, currentFieldName); + if (context.dynamic() == ObjectMapper.Dynamic.FALSE) { failIfMatchesRoutingPath(context, currentFieldName); // not dynamic, read everything up to end object context.parser().skipChildren(); @@ -525,15 +525,7 @@ private static void parseObjectDynamic(DocumentParserContext context, String cur if (dynamicObjectMapper instanceof NestedObjectMapper && context.isWithinCopyTo()) { throwOnCreateDynamicNestedViaCopyTo(dynamicObjectMapper, context); } - context.path().add(currentFieldName); - if (dynamicObjectMapper instanceof ObjectMapper objectMapper) { - if (objectMapper.subobjects() == false) { - context.path().setWithinLeafObject(true); - } - } - parseObjectOrField(context, dynamicObjectMapper); - context.path().setWithinLeafObject(false); - context.path().remove(); + doParseObject(context, currentFieldName, dynamicObjectMapper); } } @@ -556,24 +548,27 @@ private static void parseArray(DocumentParserContext context, String lastFieldNa parseNonDynamicArray(context, lastFieldName, lastFieldName); } } else { - if (context.dynamic() == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(context.parser().getTokenLocation(), context.parent().fullPath(), lastFieldName); - } else if (context.dynamic() == ObjectMapper.Dynamic.FALSE) { - context.parser().skipChildren(); + parseArrayDynamic(context, lastFieldName); + } + } + + private static void parseArrayDynamic(DocumentParserContext context, String currentFieldName) throws IOException { + ensureNotStrict(context, currentFieldName); + if (context.dynamic() == ObjectMapper.Dynamic.FALSE) { + context.parser().skipChildren(); + } else { + Mapper.Builder objectBuilderFromTemplate = DynamicFieldsBuilder.findTemplateBuilderForObject(context, currentFieldName); + if (objectBuilderFromTemplate == null) { + parseNonDynamicArray(context, currentFieldName, currentFieldName); } else { - Mapper.Builder objectBuilderFromTemplate = DynamicFieldsBuilder.findTemplateBuilderForObject(context, lastFieldName); - if (objectBuilderFromTemplate == null) { - parseNonDynamicArray(context, lastFieldName, lastFieldName); + Mapper objectMapperFromTemplate = objectBuilderFromTemplate.build(context.createDynamicMapperBuilderContext()); + if (parsesArrayValue(objectMapperFromTemplate)) { + context.addDynamicMapper(objectMapperFromTemplate.name(), objectBuilderFromTemplate); + context.path().add(currentFieldName); + parseObjectOrField(context, objectMapperFromTemplate); + context.path().remove(); } else { - Mapper objectMapperFromTemplate = objectBuilderFromTemplate.build(context.createDynamicMapperBuilderContext()); - if (parsesArrayValue(objectMapperFromTemplate)) { - context.addDynamicMapper(objectMapperFromTemplate.name(), objectBuilderFromTemplate); - context.path().add(lastFieldName); - parseObjectOrField(context, objectMapperFromTemplate); - context.path().remove(); - } else { - parseNonDynamicArray(context, lastFieldName, lastFieldName); - } + parseNonDynamicArray(context, currentFieldName, currentFieldName); } } } @@ -671,15 +666,13 @@ private static void parseNullValue(DocumentParserContext context, String lastFie if (mapper != null) { // TODO: passing null to an object seems bogus? parseObjectOrField(context, mapper); - } else if (context.dynamic() == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(context.parser().getTokenLocation(), context.parent().fullPath(), lastFieldName); + } else { + ensureNotStrict(context, lastFieldName); } } private static void parseDynamicValue(final DocumentParserContext context, String currentFieldName) throws IOException { - if (context.dynamic() == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(context.parser().getTokenLocation(), context.parent().fullPath(), currentFieldName); - } + ensureNotStrict(context, currentFieldName); if (context.dynamic() == ObjectMapper.Dynamic.FALSE) { failIfMatchesRoutingPath(context, currentFieldName); return; @@ -687,6 +680,12 @@ private static void parseDynamicValue(final DocumentParserContext context, Strin context.dynamic().getDynamicFieldsBuilder().createDynamicFieldFromValue(context, currentFieldName); } + private static void ensureNotStrict(DocumentParserContext context, String currentFieldName) { + if (context.dynamic() == ObjectMapper.Dynamic.STRICT) { + throw new StrictDynamicMappingException(context.parser().getTokenLocation(), context.parent().fullPath(), currentFieldName); + } + } + private static void failIfMatchesRoutingPath(DocumentParserContext context, String currentFieldName) { if (context.indexSettings().getIndexMetadata().getRoutingPaths().isEmpty()) { return; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index c3ea8b1c8d65b..3ff77e2192634 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -15,7 +15,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.Iterator; import java.util.Locale; import java.util.Map; @@ -76,14 +75,7 @@ public Mapper.Builder parse(String name, Map node, MappingParser } NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(name, parserContext.indexVersionCreated()); parseNested(name, node, builder); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String fieldName = entry.getKey(); - Object fieldNode = entry.getValue(); - if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)) { - iterator.remove(); - } - } + parseObjectFields(node, parserContext, builder); return builder; } @@ -152,18 +144,10 @@ public boolean isIncludeInParent() { return this.includeInParent.value(); } - public void setIncludeInParent(boolean includeInParent) { - this.includeInParent = Explicit.explicitBoolean(includeInParent); - } - public boolean isIncludeInRoot() { return this.includeInRoot.value(); } - public void setIncludeInRoot(boolean includeInRoot) { - this.includeInRoot = Explicit.explicitBoolean(includeInRoot); - } - public Map getChildren() { return this.mappers; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 851892c3a05e9..6bc82cff20e58 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -61,7 +61,7 @@ DynamicFieldsBuilder getDynamicFieldsBuilder() { DynamicFieldsBuilder getDynamicFieldsBuilder() { throw new UnsupportedOperationException("Cannot create dynamic fields when dynamic is set to [" + this + "]"); - }; + } /** * Get the root-level dynamic setting for a Mapping @@ -198,6 +198,12 @@ public Mapper.Builder parse(String name, Map node, MappingParser parserContext.incrementMappingObjectDepth(); // throws MapperParsingException if depth limit is exceeded Explicit subobjects = parseSubobjects(node); ObjectMapper.Builder builder = new Builder(name, subobjects); + parseObjectFields(node, parserContext, builder); + parserContext.decrementMappingObjectDepth(); + return builder; + } + + static void parseObjectFields(Map node, MappingParserContext parserContext, Builder builder) { for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String fieldName = entry.getKey(); @@ -206,8 +212,6 @@ public Mapper.Builder parse(String name, Map node, MappingParser iterator.remove(); } } - parserContext.decrementMappingObjectDepth(); - return builder; } @SuppressWarnings({ "unchecked", "rawtypes" }) From e70d774cad38a2cd8acdc8a830dd0aa81f294d15 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 22 Sep 2023 11:33:11 +0100 Subject: [PATCH 16/16] Extract RecoveryClusterStateDelayListeners (#99768) This utility class is needed in another test in serverless, so this commit extracts it to the test framework. --- .../indices/recovery/IndexRecoveryIT.java | 66 ++------------- .../RecoveryClusterStateDelayListeners.java | 80 +++++++++++++++++++ 2 files changed, 87 insertions(+), 59 deletions(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/indices/recovery/RecoveryClusterStateDelayListeners.java diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 7ba3b5814eb4b..8ba0f1364bac8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -39,7 +39,6 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.cluster.ClusterState; @@ -71,10 +70,6 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.core.AbstractRefCounted; -import org.elasticsearch.core.RefCounted; -import org.elasticsearch.core.Releasable; import org.elasticsearch.gateway.ReplicaShardAllocatorIT; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; @@ -1643,54 +1638,7 @@ public void testWaitForClusterStateToBeAppliedOnSourceNode() throws Exception { final long initialClusterStateVersion = clusterService().state().version(); - // Helper class to encapsulate the sync mechanism that delays applying cluster states on the primary node until the replica gives - // the go-ahead. - class ClusterStateSyncListeners implements Releasable { - private final Map> clusterStateBarriers = ConcurrentCollections.newConcurrentMap(); - private final SubscribableListener startRecoveryListener = new SubscribableListener<>(); - - private final CountDownLatch completeLatch = new CountDownLatch(1); - private final RefCounted refCounted = AbstractRefCounted.of(completeLatch::countDown); - private final List cleanup = new ArrayList<>(2); - - @Override - public void close() { - refCounted.decRef(); - safeAwait(completeLatch); - cleanup.forEach(Runnable::run); - clusterStateBarriers.values().forEach(l -> l.onResponse(null)); - } - - void addCleanup(Runnable runnable) { - cleanup.add(runnable); - } - - SubscribableListener getStateApplyDelayListener(long clusterStateVersion) { - assertThat(clusterStateVersion, greaterThanOrEqualTo(initialClusterStateVersion)); - if (refCounted.tryIncRef()) { - try { - return clusterStateBarriers.computeIfAbsent(clusterStateVersion, ignored -> new SubscribableListener<>()); - } finally { - refCounted.decRef(); - } - } else { - return SubscribableListener.newSucceeded(null); - } - } - - void onStartRecovery() { - Thread.yield(); - assertFalse(startRecoveryListener.isDone()); - startRecoveryListener.onResponse(null); - } - - public void delayUntilRecoveryStart(SubscribableListener listener) { - assertFalse(startRecoveryListener.isDone()); - startRecoveryListener.addListener(listener); - } - } - - try (var clusterStateSyncListeners = new ClusterStateSyncListeners()) { + try (var recoveryClusterStateDelayListeners = new RecoveryClusterStateDelayListeners(initialClusterStateVersion)) { final var primaryNodeTransportService = (MockTransportService) internalCluster().getInstance( TransportService.class, primaryNode @@ -1699,7 +1647,7 @@ public void delayUntilRecoveryStart(SubscribableListener listener) { Coordinator.COMMIT_STATE_ACTION_NAME, (handler, request, channel, task) -> { assertThat(request, instanceOf(ApplyCommitRequest.class)); - clusterStateSyncListeners.getStateApplyDelayListener(((ApplyCommitRequest) request).getVersion()) + recoveryClusterStateDelayListeners.getClusterStateDelayListener(((ApplyCommitRequest) request).getVersion()) .addListener( ActionListener.wrap(ignored -> handler.messageReceived(request, channel, task), e -> fail(e, "unexpected")) ); @@ -1719,28 +1667,28 @@ public void delayUntilRecoveryStart(SubscribableListener listener) { ), task ); - clusterStateSyncListeners.onStartRecovery(); + recoveryClusterStateDelayListeners.onStartRecovery(); } ); - clusterStateSyncListeners.addCleanup(primaryNodeTransportService::clearInboundRules); + recoveryClusterStateDelayListeners.addCleanup(primaryNodeTransportService::clearInboundRules); final var replicaClusterService = internalCluster().getInstance(ClusterService.class, replicaNode); final ClusterStateListener clusterStateListener = event -> { - final var primaryProceedListener = clusterStateSyncListeners.getStateApplyDelayListener(event.state().version()); + final var primaryProceedListener = recoveryClusterStateDelayListeners.getClusterStateDelayListener(event.state().version()); final var indexRoutingTable = event.state().routingTable().index(indexName); assertNotNull(indexRoutingTable); final var indexShardRoutingTable = indexRoutingTable.shard(0); if (indexShardRoutingTable.size() == 2 && indexShardRoutingTable.getAllInitializingShards().isEmpty() == false) { // this is the cluster state update which starts the recovery, so delay the primary node application until recovery // has started - clusterStateSyncListeners.delayUntilRecoveryStart(primaryProceedListener); + recoveryClusterStateDelayListeners.delayUntilRecoveryStart(primaryProceedListener); } else { // this is some other cluster state update, so we must let it proceed now primaryProceedListener.onResponse(null); } }; replicaClusterService.addListener(clusterStateListener); - clusterStateSyncListeners.addCleanup(() -> replicaClusterService.removeListener(clusterStateListener)); + recoveryClusterStateDelayListeners.addCleanup(() -> replicaClusterService.removeListener(clusterStateListener)); updateIndexSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1), indexName); ensureGreen(indexName); diff --git a/test/framework/src/main/java/org/elasticsearch/indices/recovery/RecoveryClusterStateDelayListeners.java b/test/framework/src/main/java/org/elasticsearch/indices/recovery/RecoveryClusterStateDelayListeners.java new file mode 100644 index 0000000000000..ba73845c7f63a --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/indices/recovery/RecoveryClusterStateDelayListeners.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.indices.recovery; + +import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.core.AbstractRefCounted; +import org.elasticsearch.core.RefCounted; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +/** + * Allows one node in an integ test to delay the application of cluster states on another node until a recovery starts, without unduly + * delaying other cluster states. The controlling node gets a listener for each cluster state version it applies using {@link + * #getClusterStateDelayListener} and either completes it straight away (when applying states unrelated to the recovery) or delays it with + * {@link #delayUntilRecoveryStart} and releases the delay with {@link #onStartRecovery()}; meanwhile the other node gets the same listener + * for each cluster state commit message and uses it to delay handling the commit. + */ +public class RecoveryClusterStateDelayListeners implements Releasable { + private final Map> clusterStateBarriers = ConcurrentCollections.newConcurrentMap(); + private final SubscribableListener startRecoveryListener = new SubscribableListener<>(); + + private final CountDownLatch completeLatch = new CountDownLatch(1); + private final RefCounted refCounted = AbstractRefCounted.of(completeLatch::countDown); + private final List cleanup = new ArrayList<>(2); + private final long initialClusterStateVersion; + + public RecoveryClusterStateDelayListeners(long initialClusterStateVersion) { + this.initialClusterStateVersion = initialClusterStateVersion; + } + + @Override + public void close() { + refCounted.decRef(); + ESTestCase.safeAwait(completeLatch); + cleanup.forEach(Runnable::run); + clusterStateBarriers.values().forEach(l -> l.onResponse(null)); + } + + void addCleanup(Runnable runnable) { + cleanup.add(runnable); + } + + SubscribableListener getClusterStateDelayListener(long clusterStateVersion) { + ESTestCase.assertThat(clusterStateVersion, greaterThanOrEqualTo(initialClusterStateVersion)); + if (refCounted.tryIncRef()) { + try { + return clusterStateBarriers.computeIfAbsent(clusterStateVersion, ignored -> new SubscribableListener<>()); + } finally { + refCounted.decRef(); + } + } else { + return SubscribableListener.newSucceeded(null); + } + } + + void onStartRecovery() { + Thread.yield(); + ESTestCase.assertFalse(startRecoveryListener.isDone()); + startRecoveryListener.onResponse(null); + } + + public void delayUntilRecoveryStart(SubscribableListener listener) { + ESTestCase.assertFalse(startRecoveryListener.isDone()); + startRecoveryListener.addListener(listener); + } +}